From 2f9f8c384d0bbcfe3ce4c46bcf52e2c1c839acbe Mon Sep 17 00:00:00 2001 From: KodeStar Date: Fri, 16 Oct 2026 11:02:50 +0100 Subject: [PATCH 1/5] Book labels: the genre tag, and the metadata.json's subtitle, publisher, language, abridged, genres and tags --- internal/metadata/absfile.go | 41 +++++++++++++++++++++-- internal/metadata/absfile_test.go | 25 ++++++++++++-- internal/metadata/genres.go | 32 ++++++++++++++++++ internal/metadata/genres_test.go | 52 ++++++++++++++++++++++++++++++ internal/metadata/metadata.go | 8 ++++- internal/metadata/metadata_test.go | 5 +-- internal/metadata/probe.go | 3 ++ 7 files changed, 159 insertions(+), 7 deletions(-) create mode 100644 internal/metadata/genres.go create mode 100644 internal/metadata/genres_test.go diff --git a/internal/metadata/absfile.go b/internal/metadata/absfile.go index c9fb78af..f8833149 100644 --- a/internal/metadata/absfile.go +++ b/internal/metadata/absfile.go @@ -44,13 +44,21 @@ var utf8BOM = []byte("\xef\xbb\xbf") // catalog checks each against its field's rules. type ABSFile struct { Title string + Subtitle string Author string // the file's authors, joined with ", " Narrator string // likewise Series []ABSSeries Published string // YYYY[-MM[-DD]] (ReleaseDate) from publishedDate, else publishedYear Description string // plain text: ABS keeps HTML - ASIN string - ISBN string + Publisher string + Language string + // Abridged is the file's abridged flag. ABS writes false for every book it + // wasn't told is abridged, so only true says anything. + Abridged bool + Genres []string // as the file lists them, trimmed, blanks left out + Tags []string // likewise + ASIN string + ISBN string } // ABSSeries is one of the book's series in an ABS file, in the file's order (ABS @@ -67,12 +75,18 @@ type ABSSeries struct { // {name, sequence}) are taken too. type absJSON struct { Title json.RawMessage `json:"title"` + Subtitle json.RawMessage `json:"subtitle"` Authors json.RawMessage `json:"authors"` Narrators json.RawMessage `json:"narrators"` Series json.RawMessage `json:"series"` PublishedYear json.RawMessage `json:"publishedYear"` PublishedDate json.RawMessage `json:"publishedDate"` Description json.RawMessage `json:"description"` + Publisher json.RawMessage `json:"publisher"` + Language json.RawMessage `json:"language"` + Abridged json.RawMessage `json:"abridged"` + Genres json.RawMessage `json:"genres"` + Tags json.RawMessage `json:"tags"` ISBN json.RawMessage `json:"isbn"` ASIN json.RawMessage `json:"asin"` } @@ -143,12 +157,20 @@ func ParseABSFile(data []byte) (*ABSFile, error) { if err := json.Unmarshal(bytes.TrimPrefix(data, utf8BOM), &j); err != nil { return nil, fmt.Errorf("%w: %w", ErrABSFileInvalid, err) } + var abridged bool + _ = json.Unmarshal(j.Abridged, &abridged) // anything but a bool says nothing out := &ABSFile{ Title: absScalar(j.Title), + Subtitle: absScalar(j.Subtitle), Author: strings.Join(absNames(j.Authors), ", "), Narrator: strings.Join(absNames(j.Narrators), ", "), Published: firstReleaseDate(absScalar(j.PublishedDate), absScalar(j.PublishedYear)), Description: htmlText(absScalar(j.Description)), + Publisher: absScalar(j.Publisher), + Language: absScalar(j.Language), + Abridged: abridged, + Genres: absStrings(j.Genres), + Tags: absStrings(j.Tags), ASIN: absScalar(j.ASIN), ISBN: absScalar(j.ISBN), } @@ -167,6 +189,21 @@ func absList(v json.RawMessage) []json.RawMessage { return items } +// absStrings is a list of strings' items, each trimmed: a blank one and an item +// that isn't a string are left out (nil for none). +func absStrings(list json.RawMessage) []string { + var out []string + for _, item := range absList(list) { + var s string + if json.Unmarshal(item, &s) == nil { + if s = strings.TrimSpace(s); s != "" { + out = append(out, s) + } + } + } + return out +} + // absNames is the names in a list of people, each a string or a {name} object. func absNames(list json.RawMessage) []string { var out []string diff --git a/internal/metadata/absfile_test.go b/internal/metadata/absfile_test.go index 38304d89..5b06650e 100644 --- a/internal/metadata/absfile_test.go +++ b/internal/metadata/absfile_test.go @@ -39,13 +39,18 @@ func TestParseABSFile(t *testing.T) { } want := &ABSFile{ Title: "With the Might of Angels (Dear America)", + Subtitle: "With the Might of Angels", Author: "Andrea Davis Pinkney", Narrator: "Channie Waites", Published: "2011", Description: "In the fall of 1955, twelve-year-old Dawn Rae Johnson learns she will be the only " + "one of her friends attending a previously all-white school.", - ASIN: "B005LXU2TA", - ISBN: "9780545437059", + Publisher: "Scholastic Audio Books", + Language: "English", + Genres: []string{"Children's Audiobooks"}, + Tags: []string{"Growing Up & Facts of Life", "Historical Fiction"}, + ASIN: "B005LXU2TA", + ISBN: "9780545437059", } if !reflect.DeepEqual(got, want) { t.Errorf("ParseABSFile =\n%+v\nwant\n%+v", got, want) @@ -79,6 +84,22 @@ func TestParseABSFileSeriesAndPeople(t *testing.T) { } } +// The label lists and the abridged flag: an item that isn't a string costs that +// item, and abridged says something only as true. +func TestParseABSFileLabels(t *testing.T) { + t.Parallel() + got, err := ParseABSFile([]byte(`{"genres": ["Fantasy", " ", 3, " Epic "], "tags": "not a list", "abridged": true}`)) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(got.Genres, []string{"Fantasy", "Epic"}) || got.Tags != nil || !got.Abridged { + t.Errorf("labels = %q / %q, abridged %v", got.Genres, got.Tags, got.Abridged) + } + if got, _ := ParseABSFile([]byte(`{"abridged": "yes"}`)); got.Abridged { + t.Error("a string abridged should say nothing") + } +} + // A value of the wrong type costs that value, not the file. func TestParseABSFileLenient(t *testing.T) { t.Parallel() diff --git a/internal/metadata/genres.go b/internal/metadata/genres.go new file mode 100644 index 00000000..104e7657 --- /dev/null +++ b/internal/metadata/genres.go @@ -0,0 +1,32 @@ +package metadata + +import "strings" + +// placeholderGenres are the genre tags that say only that the file is an +// audiobook (or spoken audio at all), which every book in a library is: the +// genre a ripping tool or a store writes when it has no real one. They are +// dropped, so a book's genres are the ones that tell it from another book. +var placeholderGenres = map[string]bool{ + "audiobook": true, "audiobooks": true, "audio book": true, "audio books": true, + "speech": true, "spoken word": true, "other": true, +} + +// SplitGenres reads a genre tag as a list: one tag often holds several, joined +// with ";" or "," (a tagger's own joiner) or NUL (ID3v2.4's separator between +// values, where a reader leaves it in). Each is trimmed with its inner spacing +// collapsed; blanks, the placeholders (placeholderGenres, any case) and a +// repeat of an earlier one (any case) are left out. Nil for none. +func SplitGenres(v string) []string { + var out []string + seen := map[string]bool{} + for _, part := range strings.FieldsFunc(v, func(r rune) bool { return r == ';' || r == ',' || r == 0 }) { + g := strings.Join(strings.Fields(part), " ") + key := strings.ToLower(g) + if g == "" || placeholderGenres[key] || seen[key] { + continue + } + seen[key] = true + out = append(out, g) + } + return out +} diff --git a/internal/metadata/genres_test.go b/internal/metadata/genres_test.go new file mode 100644 index 00000000..fda41484 --- /dev/null +++ b/internal/metadata/genres_test.go @@ -0,0 +1,52 @@ +package metadata + +import ( + "reflect" + "testing" +) + +func TestSplitGenres(t *testing.T) { + t.Parallel() + for _, tc := range []struct { + in string + want []string + }{ + {"", nil}, + {"Fantasy", []string{"Fantasy"}}, + {"Fantasy;Epic", []string{"Fantasy", "Epic"}}, + {" Science Fiction , Space Opera ;; ", []string{"Science Fiction", "Space Opera"}}, + {"Mystery\x00Thriller", []string{"Mystery", "Thriller"}}, + // The placeholders say only that it is an audiobook, in any case. + {"Audiobook", nil}, + {"AUDIOBOOKS; Audio Book; speech; Spoken Word; other", nil}, + {"Audiobook;Fantasy", []string{"Fantasy"}}, + // A genre once, its first spelling kept. + {"Fantasy; fantasy; FANTASY; Horror", []string{"Fantasy", "Horror"}}, + // A word that merely contains a placeholder is a genre. + {"Other Worlds", []string{"Other Worlds"}}, + } { + if got := SplitGenres(tc.in); !reflect.DeepEqual(got, tc.want) { + t.Errorf("SplitGenres(%q) = %q, want %q", tc.in, got, tc.want) + } + } +} + +// TestGenresFromTagsAndProbe: the genre tag is read by both readers, ffprobe's +// over dhowden's when it gives any (as for the other tags), and a placeholder +// from one doesn't hide the other's real genres. +func TestGenresFromTagsAndProbe(t *testing.T) { + t.Parallel() + m := &Metadata{} + applyTags(m, fakeTags{genre: "Audiobook;Fantasy"}) + if !reflect.DeepEqual(m.Genres, []string{"Fantasy"}) { + t.Fatalf("tags: Genres = %q, want [Fantasy]", m.Genres) + } + applyProbe(m, &probeResult{Tags: map[string]string{"genre": "Audiobook"}}) + if !reflect.DeepEqual(m.Genres, []string{"Fantasy"}) { + t.Errorf("a placeholder from ffprobe replaced the tag's genres: %q", m.Genres) + } + applyProbe(m, &probeResult{Tags: map[string]string{"genre": "Fantasy; Epic"}}) + if !reflect.DeepEqual(m.Genres, []string{"Fantasy", "Epic"}) { + t.Errorf("probe: Genres = %q, want [Fantasy Epic]", m.Genres) + } +} diff --git a/internal/metadata/metadata.go b/internal/metadata/metadata.go index eb34f003..d6270c63 100644 --- a/internal/metadata/metadata.go +++ b/internal/metadata/metadata.go @@ -50,7 +50,10 @@ type Metadata struct { // Released is the date a tag gives, as YYYY[-MM[-DD]] ("" when none or not a // real date; see ReleaseDate): usually the recording's release, not the work's // first publication. - Released string `json:"released,omitempty"` + Released string `json:"released,omitempty"` + // Genres are the genre tag's genres (SplitGenres: several in one tag, the + // placeholders such as "Audiobook" left out); nil for none. + Genres []string `json:"genres,omitempty"` Duration float64 `json:"duration"` Format string `json:"format"` Codec string `json:"codec"` // audio codec from ffprobe (e.g. aac, mp3, ac3) @@ -120,6 +123,9 @@ func applyTags(m *Metadata, md tag.Metadata) { if c := strings.TrimSpace(md.Composer()); c != "" { m.Narrator = c } + if g := SplitGenres(md.Genre()); g != nil { + m.Genres = g + } if raw := md.Raw(); raw != nil { if v := rawString(raw, "series", "©grp", "show", "TXXX:SERIES"); v != "" { m.Series = strings.TrimSpace(v) diff --git a/internal/metadata/metadata_test.go b/internal/metadata/metadata_test.go index ce99e2e6..882f1cd5 100644 --- a/internal/metadata/metadata_test.go +++ b/internal/metadata/metadata_test.go @@ -206,8 +206,8 @@ func TestNormalizeTagsAndRawString(t *testing.T) { // accessors applyTags reads are overridden; the rest are never called. type fakeTags struct { tag.Metadata - album, title, albumArtist, artist, composer string - raw map[string]interface{} + album, title, albumArtist, artist, composer, genre string + raw map[string]interface{} } func (f fakeTags) Album() string { return f.album } @@ -215,6 +215,7 @@ func (f fakeTags) Title() string { return f.title } func (f fakeTags) AlbumArtist() string { return f.albumArtist } func (f fakeTags) Artist() string { return f.artist } func (f fakeTags) Composer() string { return f.composer } +func (f fakeTags) Genre() string { return f.genre } func (f fakeTags) Picture() *tag.Picture { return nil } func (f fakeTags) Raw() map[string]interface{} { return f.raw } diff --git a/internal/metadata/probe.go b/internal/metadata/probe.go index 312cc5d1..0ae8313b 100644 --- a/internal/metadata/probe.go +++ b/internal/metadata/probe.go @@ -144,6 +144,9 @@ func applyProbe(m *Metadata, p *probeResult) { if v := firstReleaseDate(t["date"], t["year"]); v != "" { m.Released = v } + if g := SplitGenres(t["genre"]); g != nil { + m.Genres = g // ffprobe reads every container's genre, as for the other tags + } if v := t["series-part"]; v != "" { if start, end, ok := ParseSeriesPosition(v); ok { m.SeriesIndex, m.SeriesIndexEnd = start, end // a range: an omnibus, "1-3" From efc618ffdfc7ab9a538f81af55c65d3618103e22 Mon Sep 17 00:00:00 2001 From: KodeStar Date: Fri, 16 Oct 2026 11:02:51 +0100 Subject: [PATCH 2/5] Book labels: subtitle, publisher, language, abridged, genres and tags as layered fields; genre/tag filters, search, bulk add/remove, and the export's paths (0049) --- CLAUDE.md | 59 ++- internal/api/handlers_catalog.go | 28 +- internal/api/handlers_export.go | 18 +- internal/api/handlers_labels_test.go | 170 ++++++++ internal/api/handlers_library.go | 3 + internal/catalog/absfile.go | 14 +- internal/catalog/adminbooks.go | 96 ++++- internal/catalog/adminbooks_test.go | 3 +- internal/catalog/books.go | 51 ++- internal/catalog/export.go | 72 +++- internal/catalog/export_test.go | 75 +++- internal/catalog/labels.go | 298 ++++++++++++++ internal/catalog/labels_test.go | 387 ++++++++++++++++++ internal/catalog/model.go | 12 + internal/catalog/overrides.go | 123 +++++- internal/catalog/seriesrange_test.go | 2 +- internal/library/labels_test.go | 94 +++++ internal/library/scanner.go | 29 +- internal/matchrun/plan.go | 14 +- internal/matchrun/plan_test.go | 38 ++ internal/store/migration_0049_test.go | 61 +++ .../store/migrations/0049_book_labels.sql | 54 +++ 22 files changed, 1607 insertions(+), 94 deletions(-) create mode 100644 internal/api/handlers_labels_test.go create mode 100644 internal/catalog/labels.go create mode 100644 internal/catalog/labels_test.go create mode 100644 internal/library/labels_test.go create mode 100644 internal/store/migration_0049_test.go create mode 100644 internal/store/migrations/0049_book_labels.sql diff --git a/CLAUDE.md b/CLAUDE.md index 62f1c495..a8b0658f 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -249,7 +249,13 @@ metadata.json below). `0047` adds `bridge_jobs` (the Bridge's jobs: path-keyed, open states) and the one-row `bridge_status` (the pairing code's hash and expiry, who paired the Bridge and when, its last heartbeat); see The Bridge below. `0048` adds `books.attachments_sig` / `attachments_indexed_at` and `book_attachments` (book_id -> books CASCADE: the Bridge's checked `.audiosilo/` characters, recaps and -provenance; derived from the files like `abs_file`, so rebuildable and never carried by a move). +provenance; derived from the files like `abs_file`, so rebuildable and never carried by a move). `0049` +adds `books.subtitle` / `publisher` / `language` / `abridged` (NULL unknown) / `genres` / `tags` (effective +values; the lists canonical JSON), `books.genres_checked` (the genre tag's one-time backfill, as 0037's +`released_checked`), `book_labels` (book_id -> books CASCADE, kind `genre`|`tag`, `label_key`, the +book's spelling: derived, rebuilt by `refreshEffective`), recreates `books_fts` with `genres, tags` +columns (rows copied) and sets every `abs_indexed_at` NULL, so the next scan reads each metadata.json +again for the new fields; see Labels below. Routes `/me/queue`, `/me/collections/**`, `/me/share-targets` (`handlers_queue.go`, `handlers_collections.go`; a stranger's collection id is 404, a viewer's write 403 @@ -845,7 +851,9 @@ future metadata site can attach enrichment without reshaping the schema. The export read the row, so they see edits with no join. The player's book JSON carries `published` on every book and `description` only on `GET /libraries/{id}/item` (api's `itemBook` adds it; `catalog.Book` keeps it `json:"-"` so list, search and recent pages - stay small); `has_cover` and per-file codec are admin-only (`json:"-"`). Validation: `normalizeOverride`; sources: a scanned value is `path` when + stay small); `has_cover` and per-file codec are admin-only (`json:"-"`). Every player book (list, + search and item alike, all `omitempty`) also carries `subtitle`, `publisher`, `language`, + `abridged` (a bool, absent while unknown) and `genres` / `tags` (lists). Validation: `normalizeOverride`; sources: a scanned value is `path` when it equals what `DeriveFromPath` (or, path-first, `FromPathLayout`) yields, else `tag`; one an ABS file gives is `abs`; an override is `edited` or `community`; an enrichment-attached ASIN/ISBN reads as `community`. Revert = delete the @@ -858,12 +866,13 @@ future metadata site can attach enrichment without reshaping the schema. The off for a library; never for a file book, nor a loose file at the root) is a layer OVER the scanned values, tags and path alike, path-first libraries included (ABS's own default ranks it above the tags): `bookLayers.overABS`, in `scannedFields` before `titlePosition`. `metadata.ParseABSFile` is - lenient (a wrongly typed value costs that value; <= 4 MiB): title (no subtitle), authors and + lenient (a wrongly typed value costs that value; <= 4 MiB): title, subtitle, authors and narrators joined with ", " (strings or `{name}`), series `"Name #seq"` (the first is the main series with its position, the rest `more_series`; a series without a position keeps the tag's only while it is the same series, by `seriesKey`), `publishedDate` else `publishedYear` as `published`, the - HTML description as plain text, asin, isbn; genres, tags, publisher, language and chapters are not - read. An empty value says nothing (never clears a tag's). `absFields` stores each as an edit of its + HTML description as plain text, publisher, language, `abridged` (only `true` says anything: ABS + writes false for every book nobody marked), genres and tags (lists of strings, a non-string item + dropped; `CleanLabels`), asin, isbn; chapters are not read. An empty value says nothing (never clears a tag's). `absFields` stores each as an edit of its field would (`CleanOverride`; a refused value left out), so an ABS ASIN makes a book matched (Health, `/meta`, the community chapter check, a refresh run). `books.abs_file` holds those fields (JSON, `''` for none), `abs_sig` the file's signature (`metadata.ABSSignature`: size + mtime ns, `''` for @@ -966,6 +975,31 @@ future metadata site can attach enrichment without reshaping the schema. The chapter titles) and re-resolves them in batches of 250, flushed every 500 during the scan and on a Stop; no re-index. Until then `bookLayers.pathBaseline` resolves a snapshot against the baseline it was read with, so a path-first library still drops the folder as a series. + **Labels** (`catalog/labels.go`, migration `0049`): `genres` and `tags` are override fields whose + value is a canonical JSON list (`""` for none; `normalizeLabels`: trimmed, spacing collapsed, each + label once by `labelKey` (lowercased, spacing aside) keeping the first spelling, <= 64 characters, + <= 50 labels; an edit over a bound is a 400, a file's list is cleaned leniently instead, + `CleanLabels`). Genres come from the genre tag (`metadata.SplitGenres`: split on `;`, `,` and NUL, + placeholders such as "Audiobook", "Speech", "Spoken Word" or "Other" dropped; dhowden's then + ffprobe's, ffprobe's when it gives any, in the `scanned` snapshot) and the ABS file; tags only from + the ABS file and edits. `subtitle`, `publisher`, `language` (<= 64 characters) and `abridged` + (`"true"`/`"false"`/`""`, stored 1/0/NULL) are plain override fields from the ABS file, a match + (`matchrun.Propose`: the work's subtitle and language, the recording's publisher, abridged `true` + only when the recording is marked so; community genres are not offered, a match candidate carries + none) or an edit. `refreshEffective` writes the lists to `books.genres`/`tags`, to `book_labels` by + key (`writeLabels`, beside `keyOtherSeries`) and to the FTS `genres`/`tags` columns (a label a + line), so a search finds a book by a label. `labelFilter` is the player's `GET + /libraries/{id}/books?genre=&tag=` (one value each, any case; both narrow) and the admin + `BookFilter.Genre`/`Tag` (`?genre=`/`?tag=`); `BookFacets` counts `genres`/`tags` (the 100 most + common by key, each named by the spelling most of its books use). A bulk edit (and the book PATCH) + takes `add` / `remove` (`{"tags": ["Owned"]}`: field -> labels) besides `set` / `revert`: + `labelSets` merges them into each book's own EFFECTIVE list and writes the result as an override of + the field only where it changes the list (adding a tag a book has edits nothing on it); a field + can't be set or reverted and added to or removed from at once, nor one label added and removed. + Books indexed before `0049` get their genres from the next scan (`genres_checked`, the primary + file read once beside 0037's date read, `Catalog.SetGenres` writing the snapshot and re-resolving, + flushed with the path readings) and the ABS fields from its re-read (`abs_indexed_at` NULL), no + re-index. Path-keyed like every override, so moves and joins carry them unchanged. - **Series spellings (`catalog/moreseries.go`)**: a series is every name with one `seriesKey` (`match.SeriesKey`, the rails' key: case, accents, punctuation and spacing aside; a name with no letter or number is its own key), so "He Who Fights With Monsters", @@ -1798,10 +1832,14 @@ future metadata site can attach enrichment without reshaping the schema. The as `audiosilo--.json` - the `{"format":"audiosilo-books", "version":1,…}` envelope the community metadata site imports on its Watching page so a user can mark which entries of a series they own. **The file leaves the - server**, so it carries bibliographic facts only (title, authors, narrators, - series + position (`series_position` "1-3" for a range: the importer's grammar takes one), asin/isbn, runtime_min, chapters) and **never** a path, size, + server**, so by default it carries bibliographic facts only (title, subtitle, authors, narrators, + series + position (`series_position` "1-3" for a range: the importer's grammar takes one), + publisher, language, abridged, genres, tags, asin/isbn, runtime_min, chapters; `version` stays 1, + the meta site's readers ignore keys they don't know) and **never by default** a path, size, codec, format or anything else about the filesystem - a regression test asserts - that. `catalog.ExportLibraryBooks` composes the envelope (keyset paging over + that. `?include=paths` (`catalog.ExportOptions.IncludePaths`; any other `include` is a 400) adds + each row's `library_id` and `paths` (every copy collapsed into the row, index order) for the + admin's own scripts; the console's export menu says the file is then not for a public issue. `catalog.ExportLibraryBooks` composes the envelope (keyset paging over `ListBooks`, never OFFSET; copies of the same book within the library collapse on `exposedDedupKey`, the kept entry taking each fact from whichever copy has it - `mergeExportBook`, since the copy that sorts first may be the untagged rip); the @@ -1984,7 +2022,10 @@ The player's browse lists are `GET /libraries/{id}/authors` (`{authors, unknown} granted books; `api/handlers_browse.go`), and `GET /libraries/{id}/books` filters by `series=` (any spelling, see Series spellings), and by `author=` / `narrator=` matching the whole credit or exactly one person it names (`catalog.creditFilter`: an FTS5 phrase narrows the candidates, -the `credit_has` SQL function keeps the exact ones; `/admin/books` filters the same way). +the `credit_has` SQL function keeps the exact ones; `/admin/books` filters the same way), +and by `genre=` / `tag=` (one value each, any case, `catalog.labelFilter`; see Labels; an older +server ignores them, so a player filtering a loaded library by the books' own `genres` / `tags` +needs no capability). Books in several series (capability `series_memberships`): with `memberships=1`, `/books?series=` also matches a book whose `more_series` names it (`catalog.seriesFilter`, as `/admin/books`) and `/series` counts a book in every series it is in (`extra_books`); diff --git a/internal/api/handlers_catalog.go b/internal/api/handlers_catalog.go index 7899c317..5ebb4f2f 100644 --- a/internal/api/handlers_catalog.go +++ b/internal/api/handlers_catalog.go @@ -82,7 +82,8 @@ func (a *API) handleAdminBookFacets(w http.ResponseWriter, r *http.Request) { func bookFilterFromQuery(q url.Values) (catalog.BookFilter, string) { f := catalog.BookFilter{ Query: strings.TrimSpace(q.Get("q")), Author: q.Get("author"), Series: q.Get("series"), - Narrator: q.Get("narrator"), Formats: q["format"], Codecs: q["codec"], + Narrator: q.Get("narrator"), Genre: strings.TrimSpace(q.Get("genre")), Tag: strings.TrimSpace(q.Get("tag")), + Formats: q["format"], Codecs: q["codec"], } var ok bool if f.LibraryID, ok = parseOptionalID(q.Get("library_id")); !ok { @@ -261,9 +262,14 @@ func (a *API) handleCheckCommunityChapters(w http.ResponseWriter, r *http.Reques // editRequest is the body of a book edit (PATCH) and, without chapters, of a bulk // edit. type editRequest struct { - Set map[string]string `json:"set"` - Revert []string `json:"revert"` - Source string `json:"source"` + Set map[string]string `json:"set"` + Revert []string `json:"revert"` + // Add and Remove change a label list (genres, tags) by name instead of + // setting it, so a bulk edit keeps each book's own labels + // (catalog.BookEdit.Add). + Add map[string][]string `json:"add"` + Remove map[string][]string `json:"remove"` + Source string `json:"source"` Chapters *struct { Set map[int]string `json:"set"` Revert []int `json:"revert"` @@ -274,7 +280,8 @@ type editRequest struct { } func (e editRequest) toEdit(userID int64) catalog.BookEdit { - edit := catalog.BookEdit{Set: e.Set, Revert: e.Revert, Source: e.Source, ChapterSource: e.ChapterSource, UserID: userID} + edit := catalog.BookEdit{Set: e.Set, Revert: e.Revert, Add: e.Add, Remove: e.Remove, Source: e.Source, + ChapterSource: e.ChapterSource, UserID: userID} if e.Chapters != nil { edit.ChapterSet, edit.ChapterRevert = e.Chapters.Set, e.Chapters.Revert } @@ -282,7 +289,8 @@ func (e editRequest) toEdit(userID int64) catalog.BookEdit { } // details are the edit for the audit log: the values set (a long one cut short), -// the fields reverted, where they came from, and how many chapter titles changed. +// the fields reverted, the labels added and removed, where they came from, and +// how many chapter titles changed. func (e editRequest) details() map[string]any { d := map[string]any{} if len(e.Set) > 0 { @@ -298,6 +306,12 @@ func (e editRequest) details() map[string]any { if len(e.Revert) > 0 { d["revert"] = e.Revert } + if len(e.Add) > 0 { + d["add"] = e.Add + } + if len(e.Remove) > 0 { + d["remove"] = e.Remove + } if e.Source != "" { d["source"] = e.Source } @@ -313,7 +327,7 @@ func (e editRequest) details() map[string]any { } func (e editRequest) empty() bool { - return len(e.Set) == 0 && len(e.Revert) == 0 && e.ChapterSource == "" && + return len(e.Set) == 0 && len(e.Revert) == 0 && len(e.Add) == 0 && len(e.Remove) == 0 && e.ChapterSource == "" && (e.Chapters == nil || (len(e.Chapters.Set) == 0 && len(e.Chapters.Revert) == 0)) } diff --git a/internal/api/handlers_export.go b/internal/api/handlers_export.go index 60456744..7b06e194 100644 --- a/internal/api/handlers_export.go +++ b/internal/api/handlers_export.go @@ -19,15 +19,27 @@ import ( // straight to the response with an encoder rather than buffering a string of // its own. // -// The file carries bibliographic facts only - no paths, sizes, codecs or anything -// else describing the filesystem - because it is meant to leave the server. +// By default the file carries bibliographic facts only - no paths, sizes, codecs +// or anything else describing the filesystem - because it is meant to leave the +// server. ?include=paths adds each row's library id and paths, for the admin's +// own scripts (catalog.ExportOptions); any other include is a 400, so a typo +// never silently drops what was asked for. func (a *API) handleExportLibrary(w http.ResponseWriter, r *http.Request) { id, ok := pathInt(r, "id") if !ok { writeError(w, http.StatusBadRequest, "invalid library id") return } - exp, err := a.cat.ExportLibraryBooks(r.Context(), id, Version) + var opt catalog.ExportOptions + switch r.URL.Query().Get("include") { + case "": + case "paths": + opt.IncludePaths = true + default: + writeError(w, http.StatusBadRequest, `include must be "paths"`) + return + } + exp, err := a.cat.ExportLibraryBooks(r.Context(), id, Version, opt) if errors.Is(err, catalog.ErrNotFound) { writeError(w, http.StatusNotFound, "library not found") return diff --git a/internal/api/handlers_labels_test.go b/internal/api/handlers_labels_test.go new file mode 100644 index 00000000..40e32544 --- /dev/null +++ b/internal/api/handlers_labels_test.go @@ -0,0 +1,170 @@ +package api + +import ( + "context" + "encoding/json" + "net/http" + "slices" + "strconv" + "strings" + "testing" + + "github.com/kodestar/audiosilo-server/internal/catalog" +) + +// TestBookLabelsAPI: a bulk edit adds a tag to each book's own list and takes +// one away; the player's /books filters by tag and genre (any case), /search +// finds a book by a tag, /item and the list carry the new fields, and the admin +// list and facets read them too. +func TestBookLabelsAPI(t *testing.T) { + t.Parallel() + e := newTestEnv(t) + adminTok, _, _ := adminAndMember(t, e) + libID, base := seedCatalog(t, e) + lib := strconv.FormatInt(libID, 10) + ctx := context.Background() + if err := e.cat.EditBook(ctx, libID, "Andy Weir/The Martian", catalog.BookEdit{Set: map[string]string{ + catalog.FieldTags: `["Keep"]`, catalog.FieldGenres: `["Science Fiction"]`, catalog.FieldSubtitle: "A Novel", + catalog.FieldPublisher: "Podium", catalog.FieldLanguage: "English", catalog.FieldAbridged: "false", + }}); err != nil { + t.Fatal(err) + } + both := `[{"library_id":` + lib + `,"path":"Andy Weir/The Martian"},{"library_id":` + lib + `,"path":"Andy Weir/Artemis"}]` + if resp, body := e.do(t, "POST", "/api/v1/admin/books/bulk", adminTok, + `{"books":`+both+`,"add":{"tags":["Owned","Read"]}}`); resp.StatusCode != 200 { + t.Fatalf("bulk add = %d %s", resp.StatusCode, body) + } + if resp, body := e.do(t, "POST", "/api/v1/admin/books/bulk", adminTok, + `{"books":`+both+`,"remove":{"tags":["read"]}}`); resp.StatusCode != 200 { + t.Fatalf("bulk remove = %d %s", resp.StatusCode, body) + } + if resp, body := e.do(t, "POST", "/api/v1/admin/books/bulk", adminTok, + `{"books":`+both+`,"set":{"tags":"[]"},"add":{"tags":["x"]}}`); resp.StatusCode != http.StatusBadRequest { + t.Errorf("set and add of one field = %d %s, want 400", resp.StatusCode, body) + } + + books := func(query string) []string { + t.Helper() + resp, body := e.do(t, "GET", "/api/v1/libraries/"+lib+"/books?"+query, adminTok, "") + if resp.StatusCode != 200 { + t.Fatalf("books?%s = %d %s", query, resp.StatusCode, body) + } + var page struct { + Books []struct { + Path string `json:"rel_path"` + Tags []string `json:"tags"` + } `json:"books"` + } + if err := json.Unmarshal([]byte(body), &page); err != nil { + t.Fatal(err) + } + var out []string + for _, b := range page.Books { + out = append(out, b.Path+":"+strings.Join(b.Tags, ",")) + } + slices.Sort(out) + return out + } + if got, want := books("tag=OWNED"), []string{"Andy Weir/Artemis:Owned", "Andy Weir/The Martian:Keep,Owned"}; !slices.Equal(got, want) { + t.Errorf("tag=OWNED = %v, want %v", got, want) + } + if got, want := books("genre=science%20fiction&tag=keep"), []string{"Andy Weir/The Martian:Keep,Owned"}; !slices.Equal(got, want) { + t.Errorf("genre and tag = %v, want %v", got, want) + } + if got := books("tag=read"); got != nil { + t.Errorf("tag=read = %v, want none (removed)", got) + } + + _, body := e.do(t, "GET", "/api/v1/search?q=keep", adminTok, "") + if !strings.Contains(body, `"rel_path":"Andy Weir/The Martian"`) || strings.Contains(body, "Artemis") { + t.Errorf("search by tag = %s", body) + } + + resp, body := e.do(t, "GET", "/api/v1/libraries/"+lib+"/item?path="+escape("Andy Weir/The Martian"), adminTok, "") + if resp.StatusCode != 200 { + t.Fatalf("item = %d %s", resp.StatusCode, body) + } + var item map[string]json.RawMessage + if err := json.Unmarshal([]byte(body), &item); err != nil { + t.Fatal(err) + } + for key, want := range map[string]string{ + "subtitle": `"A Novel"`, "publisher": `"Podium"`, "language": `"English"`, "abridged": `false`, + "genres": `["Science Fiction"]`, "tags": `["Keep","Owned"]`, + } { + if got := string(item[key]); got != want { + t.Errorf("item %s = %s, want %s", key, got, want) + } + } + // Artemis has none of them: the keys are absent, abridged included (unknown). + _, body = e.do(t, "GET", "/api/v1/libraries/"+lib+"/item?path="+escape("Andy Weir/Artemis"), adminTok, "") + for _, key := range []string{`"subtitle"`, `"publisher"`, `"language"`, `"abridged"`, `"genres"`} { + if strings.Contains(body, key) { + t.Errorf("Artemis item carries %s: %s", key, body) + } + } + + _, body = e.do(t, "GET", "/api/v1/admin/books?tag=keep", adminTok, "") + if !strings.Contains(body, `"tags":["Keep","Owned"]`) || !strings.Contains(body, `"subtitle":"A Novel"`) || strings.Contains(body, "Artemis") { + t.Errorf("admin list tag=keep = %s", body) + } + _, body = e.do(t, "GET", "/api/v1/admin/books/facets?library_id="+lib, adminTok, "") + if !strings.Contains(body, `"tags":[{"value":"Owned","count":2},{"value":"Keep","count":1}]`) || + !strings.Contains(body, `"genres":[{"value":"Science Fiction","count":1}]`) { + t.Errorf("facets = %s", body) + } + _, body = e.do(t, "GET", base+"/book?path="+escape("Andy Weir/Artemis"), adminTok, "") + if !strings.Contains(body, `"tags":{"value":"[\"Owned\"]","source":"edited"`) { + t.Errorf("book page tags provenance = %s", body) + } +} + +// TestExportIncludePaths: the allowed and denied pair for ?include=paths: an +// admin's export names each row's library and paths only when asked, a member is +// refused either way, and an unknown include is a 400. +func TestExportIncludePaths(t *testing.T) { + t.Parallel() + e := newTestEnv(t) + adminTok, memberTok, _ := adminAndMember(t, e) + libID, _ := seedCatalog(t, e) + path := "/api/v1/admin/libraries/" + strconv.FormatInt(libID, 10) + "/export" + + resp, body := e.do(t, "GET", path+"?include=paths", adminTok, "") + if resp.StatusCode != 200 { + t.Fatalf("export with paths = %d %s", resp.StatusCode, body) + } + var out struct { + Books []struct { + Title string `json:"title"` + LibraryID int64 `json:"library_id"` + Paths []string `json:"paths"` + } `json:"books"` + } + if err := json.Unmarshal([]byte(body), &out); err != nil { + t.Fatal(err) + } + got := map[string][]string{} + for _, b := range out.Books { + if b.LibraryID != libID { + t.Errorf("%s: library_id = %d, want %d", b.Title, b.LibraryID, libID) + } + got[b.Title] = b.Paths + } + if !slices.Equal(got["The Martian"], []string{"Andy Weir/The Martian"}) || !slices.Equal(got["Artemis"], []string{"Andy Weir/Artemis"}) { + t.Errorf("paths = %v", got) + } + + _, body = e.do(t, "GET", path, adminTok, "") + if strings.Contains(body, `"paths"`) || strings.Contains(body, `"library_id"`) || strings.Contains(body, "Andy Weir/") { + t.Errorf("the default export carries paths: %s", body) + } + if resp, _ := e.do(t, "GET", path+"?include=paths", memberTok, ""); resp.StatusCode != http.StatusForbidden { + t.Errorf("member export with paths = %d, want 403", resp.StatusCode) + } + if resp, _ := e.do(t, "GET", path+"?include=paths", "", ""); resp.StatusCode != http.StatusUnauthorized { + t.Errorf("anonymous export with paths = %d, want 401", resp.StatusCode) + } + if resp, _ := e.do(t, "GET", path+"?include=files", adminTok, ""); resp.StatusCode != http.StatusBadRequest { + t.Errorf("unknown include = %d, want 400", resp.StatusCode) + } +} diff --git a/internal/api/handlers_library.go b/internal/api/handlers_library.go index 52b11a12..f8b8a31c 100644 --- a/internal/api/handlers_library.go +++ b/internal/api/handlers_library.go @@ -7,6 +7,7 @@ import ( "net/http" "os" "strconv" + "strings" "time" "github.com/kodestar/audiosilo-server/internal/auth" @@ -298,6 +299,8 @@ func (a *API) handleListBooks(w http.ResponseWriter, r *http.Request) { Author: r.URL.Query().Get("author"), Series: r.URL.Query().Get("series"), Narrator: r.URL.Query().Get("narrator"), + Genre: strings.TrimSpace(r.URL.Query().Get("genre")), + Tag: strings.TrimSpace(r.URL.Query().Get("tag")), Memberships: r.URL.Query().Get("memberships") == "1", Sort: r.URL.Query().Get("sort"), Limit: queryInt(r, "limit", 50), diff --git a/internal/catalog/absfile.go b/internal/catalog/absfile.go index adaa1852..6adaadcd 100644 --- a/internal/catalog/absfile.go +++ b/internal/catalog/absfile.go @@ -29,8 +29,10 @@ type ABSReading struct { } // absFields is what an ABS file says as stored field values: each as an edit of -// that field would store it, a value the field refuses left out. The first -// series is the book's, with its position; the others are its more_series. +// that field would store it, a value the field refuses left out (and a label the +// list refuses, CleanLabels). The first series is the book's, with its +// position; the others are its more_series. abridged is "true" only: ABS +// writes false for every book nobody marked, so false says nothing. func absFields(f *metadata.ABSFile) bookFields { out := bookFields{} if f == nil { @@ -42,9 +44,17 @@ func absFields(f *metadata.ABSFile) bookFields { } } set(FieldTitle, f.Title) + set(FieldSubtitle, f.Subtitle) set(FieldAuthor, f.Author) set(FieldNarrator, f.Narrator) set(FieldPublished, f.Published) + set(FieldPublisher, f.Publisher) + set(FieldLanguage, f.Language) + if f.Abridged { + set(FieldAbridged, "true") + } + set(FieldGenres, EncodeLabels(CleanLabels(f.Genres))) + set(FieldTags, EncodeLabels(CleanLabels(f.Tags))) set(FieldDescription, f.Description) set(FieldASIN, f.ASIN) set(FieldISBN, f.ISBN) diff --git a/internal/catalog/adminbooks.go b/internal/catalog/adminbooks.go index f7287624..8e0af22f 100644 --- a/internal/catalog/adminbooks.go +++ b/internal/catalog/adminbooks.go @@ -32,6 +32,7 @@ type AdminBook struct { Path string `json:"path"` IsFolder bool `json:"is_folder"` Title string `json:"title"` + Subtitle string `json:"subtitle"` Author string `json:"author"` Narrator string `json:"narrator"` // Authors and Narrators are the people Author and Narrator name (creditNames; @@ -48,6 +49,13 @@ type AdminBook struct { SeriesList []SeriesRef `json:"series_list"` moreSeries string Published string `json:"published"` + Publisher string `json:"publisher"` + Language string `json:"language"` + // Abridged is null when nothing says (Book.Abridged). + Abridged *bool `json:"abridged"` + // Genres and Tags are the book's labels (labels.go), never nil. + Genres []string `json:"genres"` + Tags []string `json:"tags"` // Released is the tags' date (Book.Released), for the release-date sort. Released string `json:"-"` Duration float64 `json:"duration"` @@ -145,12 +153,13 @@ const ( // directPlayableExpr is media.DirectPlayable in SQL. var directPlayableExpr = media.DirectPlayableSQL("b.codec") -var adminBookCols = `b.id, b.library_id, l.name, b.rel_path, b.is_folder, b.title, b.author, +var adminBookCols = `b.id, b.library_id, l.name, b.rel_path, b.is_folder, b.title, b.subtitle, b.author, b.narrator, b.series, b.series_index, b.series_index_end, b.more_series, b.published, b.released, b.duration, b.format, b.codec, ` + `b.size, b.added_at, ` + customCoverExpr + `, ` + chapterCountExpr + `, ` + fileCountExpr + `, b.asin, b.isbn, ` + matchedExpr + `, ` + editedExpr + `, ` + editedFieldsExpr + `, ` + hasCoverExpr + `, ` + directPlayableExpr + `, b.scan_error, b.scan_error_file, b.scan_error_detail, COALESCE(b.suspect_parts, 0), ` + - chaptersSourceExpr + `, ` + chaptersCheckExpr + `, b.cover_art, b.cover_color` + chaptersSourceExpr + `, ` + chaptersCheckExpr + `, b.cover_art, b.cover_color, + b.publisher, b.language, b.abridged, b.genres, b.tags` // scanAdminBook reads one row of adminBookCols into b. func scanAdminBook(rows *sql.Rows, b *AdminBook) error { @@ -183,18 +192,20 @@ func creditFilter(prefix, field, value string) (string, []any) { // adminBookDest returns the scan destinations for adminBookCols, in order; // finish, called after the scan, derives what the row carries beyond its -// columns: Authors and Narrators from Author and Narrator and SeriesList from the -// series columns (never nil, so the wire carries [] for none), and CoverColor, -// when the stored colour is for the current art. +// columns: Authors and Narrators from Author and Narrator, SeriesList from the +// series columns and Genres and Tags from theirs (never nil, so the wire carries +// [] for none), and CoverColor, when the stored colour is for the current art. func adminBookDest(b *AdminBook) (dest []any, finish func()) { - var art, color string - return []any{&b.id, &b.LibraryID, &b.LibraryName, &b.Path, &b.IsFolder, &b.Title, &b.Author, + var art, color, genres, tags string + return []any{&b.id, &b.LibraryID, &b.LibraryName, &b.Path, &b.IsFolder, &b.Title, &b.Subtitle, &b.Author, &b.Narrator, &b.Series, &b.SeriesIndex, &b.SeriesIndexEnd, &b.moreSeries, &b.Published, &b.Released, &b.Duration, &b.Format, &b.Codec, &b.Size, &b.AddedAt, &b.CustomCover, &b.ChapterCount, &b.FileCount, &b.ASIN, &b.ISBN, &b.Matched, &b.Edited, &b.EditedFields, &b.HasCover, &b.DirectPlayable, &b.ScanError, &b.ScanErrorFile, &b.ScanErrorDetail, &b.SuspectParts, &b.ChaptersSource, &b.ChaptersCheck, - &art, &color}, + &art, &color, &b.Publisher, &b.Language, &b.Abridged, &genres, &tags}, func() { + b.Genres = append([]string{}, ParseLabels(genres)...) + b.Tags = append([]string{}, ParseLabels(tags)...) b.Authors = append([]string{}, creditNames(b.Author)...) b.Narrators = append([]string{}, creditNames(b.Narrator)...) b.SeriesList = SeriesList(SeriesRef{Name: b.Series, Position: b.SeriesIndex, PositionEnd: b.SeriesIndexEnd}, b.moreSeries) @@ -210,6 +221,8 @@ type BookFilter struct { Author string // the whole effective value, or exactly one person it names (creditFilter) Series string // the main series, or one more_series names, in any spelling (seriesFilter) Narrator string // as Author + Genre string // a genre the book has, in any case (labelFilter) + Tag string // likewise a tag Formats []string Codecs []string DirectPlayable *bool @@ -239,6 +252,8 @@ const ( facetHasChapters = "has_chapters" facetMatched = "matched" facetEdited = "edited" + facetGenre = "genre" + facetTag = "tag" ) // where builds the WHERE clause (over `books b`) for f, leaving out the facet @@ -269,7 +284,7 @@ func (f BookFilter) where(skip string) (string, []any) { // whole library by its library index instead (the unary + rules it out). // A value with no phrase to match isn't looked up there, so the library // index stays the way in. - if hasFTSPhrase(f.Author) || hasFTSPhrase(f.Narrator) || f.Series != "" { + if hasFTSPhrase(f.Author) || hasFTSPhrase(f.Narrator) || f.Series != "" || f.Genre != "" || f.Tag != "" { add("+b.library_id = ?", f.LibraryID) } else { add("b.library_id = ?", f.LibraryID) @@ -293,6 +308,14 @@ func (f BookFilter) where(skip string) (string, []any) { cond, a := creditFilter("b.", "narrator", f.Narrator) add(cond, a...) } + if f.Genre != "" && skip != facetGenre { + cond, a := labelFilter("b.", labelGenre, f.Genre) + add(cond, a...) + } + if f.Tag != "" && skip != facetTag { + cond, a := labelFilter("b.", labelTag, f.Tag) + add(cond, a...) + } if len(f.Formats) > 0 && skip != facetFormat { in("b.format", f.Formats) } @@ -578,8 +601,16 @@ type BookFacets struct { HasChapters BoolFacet `json:"has_chapters"` Matched BoolFacet `json:"matched"` Edited BoolFacet `json:"edited"` + // Genres and Tags are the most common labels (at most maxLabelFacets each), + // each named by the spelling most of its books use. + Genres []FacetCount `json:"genres"` + Tags []FacetCount `json:"tags"` } +// maxLabelFacets bounds each label facet: a library can hold thousands of +// tags, and the filter panel shows the most common. +const maxLabelFacets = 100 + // BookFacets computes the facet counts for f. func (c *Catalog) BookFacets(ctx context.Context, f BookFilter) (*BookFacets, error) { out := &BookFacets{} @@ -601,6 +632,15 @@ func (c *Catalog) BookFacets(ctx context.Context, f BookFilter) (*BookFacets, er return nil, err } } + for _, d := range []struct { + dim, kind string + dst *[]FacetCount + }{{facetGenre, labelGenre, &out.Genres}, {facetTag, labelTag, &out.Tags}} { + where, args := f.where(d.dim) + if *d.dst, err = c.labelFacet(ctx, d.kind, where, args); err != nil { + return nil, err + } + } bools := []struct { dim, expr string @@ -653,6 +693,44 @@ func (c *Catalog) BookFacets(ctx context.Context, f BookFilter) (*BookFacets, er return out, nil } +// labelFacet counts the books (over `books b` WHERE where) by each label of +// kind: the most common maxLabelFacets, ties alphabetical, each named by the +// spelling most of its books use (ties alphabetical too). +func (c *Catalog) labelFacet(ctx context.Context, kind, where string, args []any) ([]FacetCount, error) { + type spelled struct { + key, name string + n int + } + rows, err := queryRows(ctx, c.db, func(r *sql.Rows, s *spelled) error { return r.Scan(&s.key, &s.name, &s.n) }, + `SELECT bl.label_key, bl.name, COUNT(*) FROM book_labels bl JOIN books b ON b.id = bl.book_id + WHERE bl.kind = ? AND `+where+` GROUP BY bl.label_key, bl.name`, append([]any{kind}, args...)...) + if err != nil { + return nil, err + } + type label struct { + FacetCount + best int // the books of its most used spelling + } + byKey := map[string]*label{} + for _, r := range rows { + l := byKey[r.key] + if l == nil { + l = &label{} + byKey[r.key] = l + } + l.Count += r.n + if r.n > l.best || (r.n == l.best && r.name < l.Value) { + l.Value, l.best = r.name, r.n + } + } + out := make([]FacetCount, 0, len(byKey)) + for _, l := range byKey { + out = append(out, l.FacetCount) + } + slices.SortFunc(out, func(a, b FacetCount) int { return cmp.Or(b.Count-a.Count, compareFold(a.Value, b.Value)) }) + return out[:min(len(out), maxLabelFacets)], nil +} + // PersonCount is one author or narrator and their books. type PersonCount struct { Name string `json:"name"` diff --git a/internal/catalog/adminbooks_test.go b/internal/catalog/adminbooks_test.go index 1d3e0df7..1a4cd95a 100644 --- a/internal/catalog/adminbooks_test.go +++ b/internal/catalog/adminbooks_test.go @@ -117,7 +117,8 @@ func TestListAdminBooksRowShape(t *testing.T) { got.id, got.moreSeries = 0, "" want := AdminBook{LibraryID: libA, SeriesList: []SeriesRef{}, LibraryName: "Fiction", Path: "Herbert/Dune", Title: "Dune", Author: "Frank Herbert", Narrator: "Scott Brick", Authors: []string{"Frank Herbert"}, Narrators: []string{"Scott Brick"}, Duration: 9000, Format: "flac", Codec: "ac3", AddedAt: "2023-06-01T00:00:00Z", - HasCover: true, FileCount: 1, Edited: true, EditedFields: fieldList{FieldNarrator}, ChaptersSource: ChaptersFromFiles} + HasCover: true, FileCount: 1, Edited: true, EditedFields: fieldList{FieldNarrator}, ChaptersSource: ChaptersFromFiles, + Genres: []string{}, Tags: []string{}} if !reflect.DeepEqual(got, want) { t.Fatalf("row = %+v\nwant %+v", got, want) } diff --git a/internal/catalog/books.go b/internal/catalog/books.go index 403cc9e8..7df30e79 100644 --- a/internal/catalog/books.go +++ b/internal/catalog/books.go @@ -63,10 +63,10 @@ func (c *Catalog) UpsertBook(ctx context.Context, b *Book) (int64, error) { series_index, narrator, duration, asin, isbn, cover_path, format, codec, size, mtime, content_hash, indexed_at, added_at, published, description, has_cover, scanned, scan_error, scan_error_file, scan_error_detail, suspect_parts, split_parent, - released, released_checked, + released, released_checked, genres_checked, scanned_chapters, chapters_hash, chapters_source, chapters_fit, abs_sig, abs_indexed_at, abs_file, attachments_sig, attachments_indexed_at) - VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,1,?,?,'','',?,?,?,?,?) + VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,1,1,?,?,'','',?,?,?,?,?) ON CONFLICT(library_id, rel_path) DO UPDATE SET is_folder=excluded.is_folder, title=excluded.title, author=excluded.author, series=excluded.series, series_index=excluded.series_index, @@ -79,7 +79,7 @@ func (c *Catalog) UpsertBook(ctx context.Context, b *Book) (int64, error) { scan_error=excluded.scan_error, scan_error_file=excluded.scan_error_file, scan_error_detail=excluded.scan_error_detail, suspect_parts=excluded.suspect_parts, split_parent=excluded.split_parent, - released=excluded.released, released_checked=1, + released=excluded.released, released_checked=1, genres_checked=1, -- The rows below are the scan's: refreshEffective puts a community -- list back over them when it should. scanned_chapters=excluded.scanned_chapters, chapters_hash=excluded.chapters_hash, @@ -256,6 +256,10 @@ type Signature struct { // (indexed before migration 0037; see books.released_checked), which the next // scan reads (SetReleased). ReleasedUnchecked bool + // GenresUnchecked: a book whose genre tag hasn't been read (indexed before + // migration 0049; see books.genres_checked), which the next scan reads + // (SetGenres). + GenresUnchecked bool // PathCheck: what a book's snapshot needs from the next scan now that the path // baseline reads a lone folder as the author (pathCheckExpr; SetPathReading). PathCheck PathCheck @@ -285,7 +289,7 @@ func (c *Catalog) Signatures(ctx context.Context, libraryID int64) (map[string]S rows, err := c.db.QueryContext(ctx, `SELECT rel_path, mtime, size, duration, codec, content_hash, cover_path, has_cover, is_folder, suspect_parts IS NULL, scan_error, scan_error_file, split_parent, - released_checked = 0, `+pathCheckExpr+`, abs_sig, `+absUncheckedExpr+`, + released_checked = 0, genres_checked = 0, `+pathCheckExpr+`, abs_sig, `+absUncheckedExpr+`, attachments_sig, `+attachmentsUncheckedExpr+` FROM books WHERE library_id = ?`, libraryID) if err != nil { @@ -298,7 +302,7 @@ func (c *Catalog) Signatures(ctx context.Context, libraryID int64) (map[string]S var sig Signature if err := rows.Scan(&rel, &sig.MTime, &sig.Size, &sig.Duration, &sig.Codec, &sig.ContentHash, &sig.CoverPath, &sig.HasCover, &sig.IsFolder, &sig.SuspectUnchecked, - &sig.ScanError, &sig.ScanErrorFile, &sig.SplitParent, &sig.ReleasedUnchecked, + &sig.ScanError, &sig.ScanErrorFile, &sig.SplitParent, &sig.ReleasedUnchecked, &sig.GenresUnchecked, &sig.PathCheck, &sig.ABSSig, &sig.ABSUnchecked, &sig.AttachmentsSig, &sig.AttachmentsUnchecked); err != nil { return nil, err } @@ -484,19 +488,22 @@ func deleteBook(ctx context.Context, tx *sql.Tx, id int64) error { const bookCols = `id, library_id, rel_path, is_folder, title, author, series, series_index, series_index_end, more_series, narrator, duration, asin, isbn, cover_path, format, codec, size, mtime, - added_at, content_hash, published, has_cover, cover_art, cover_color, attachments_sig` + added_at, content_hash, published, has_cover, cover_art, cover_color, attachments_sig, + subtitle, publisher, language, abridged, genres, tags` // bookDest returns the scan destinations for bookCols, in order, so every query // selecting bookCols (plain or prefixed) scans it the same way; finish, called -// after the scan, derives the cover fields and the series list from the columns -// behind them. +// after the scan, derives the cover fields, the series list and the labels from +// the columns behind them. func bookDest(b *Book) (dest []any, finish func()) { - var more, art, color, attachments string + var more, art, color, attachments, genres, tags string return []any{&b.ID, &b.LibraryID, &b.RelPath, &b.IsFolder, &b.Title, &b.Author, &b.Series, &b.SeriesIndex, &b.SeriesIndexEnd, &more, &b.Narrator, &b.Duration, &b.ASIN, &b.ISBN, &b.CoverPath, &b.Format, &b.Codec, &b.Size, &b.MTime, &b.AddedAt, &b.ContentHash, - &b.Published, &b.HasCover, &art, &color, &attachments}, + &b.Published, &b.HasCover, &art, &color, &attachments, + &b.Subtitle, &b.Publisher, &b.Language, &b.Abridged, &genres, &tags}, func() { + b.Genres, b.Tags = ParseLabels(genres), ParseLabels(tags) b.CoverVersion = CoverVersion(art) b.Generated = generatedSig(attachments) b.CoverColor, _ = storedCoverColor(art, color) @@ -657,10 +664,14 @@ type ListOptions struct { // client that places books by series_index alone leaves it off. Memberships bool Narrator string // optional, as Author - Sort string // "author" (default) | "title" | "recent" - Limit int - Cursor string // opaque keyset cursor from a previous page - Scope *Scope // optional access scope; nil = unrestricted (admin/internal) + // Genre and Tag are optional: the books with that genre / tag, in any case + // (labelFilter). + Genre string + Tag string + Sort string // "author" (default) | "title" | "recent" + Limit int + Cursor string // opaque keyset cursor from a previous page + Scope *Scope // optional access scope; nil = unrestricted (admin/internal) } // Page is a page of books plus the cursor for the next page ("" when exhausted). @@ -714,9 +725,9 @@ func (c *Catalog) ListBooks(ctx context.Context, opt ListOptions) (*Page, error) where := []string{"library_id = ?"} args := []any{opt.LibraryID} - if hasFTSPhrase(opt.Author) || hasFTSPhrase(opt.Narrator) || opt.Series != "" { - // The full-text candidates or the series' own index drive these filters, - // not the library index (as in BookFilter.where). + if hasFTSPhrase(opt.Author) || hasFTSPhrase(opt.Narrator) || opt.Series != "" || opt.Genre != "" || opt.Tag != "" { + // The full-text candidates, the series' own index or the labels' drive + // these filters, not the library index (as in BookFilter.where). where[0] = "+library_id = ?" } if opt.Author != "" { @@ -731,6 +742,12 @@ func (c *Catalog) ListBooks(ctx context.Context, opt ListOptions) (*Page, error) cond, a := creditFilter("", "narrator", opt.Narrator) where, args = append(where, cond), append(args, a...) } + for kind, name := range map[string]string{labelGenre: opt.Genre, labelTag: opt.Tag} { + if name != "" { + cond, a := labelFilter("", kind, name) + where, args = append(where, cond), append(args, a...) + } + } // Restrict to the caller's access scope (share path rules), if provided. if opt.Scope != nil { frag, fargs := pathFilterSQL("rel_path", *opt.Scope) diff --git a/internal/catalog/export.go b/internal/catalog/export.go index d03ca2f6..214d817e 100644 --- a/internal/catalog/export.go +++ b/internal/catalog/export.go @@ -16,9 +16,14 @@ import ( // // The file is meant to LEAVE the server - a user downloads it and uploads it to // the community metadata site (meta.audiosilo.app) to mark which entries of a -// series they own. So it carries bibliographic facts only: titles, contributors, -// series position, identifiers, runtime. Never a path, size, codec, format or -// anything else that describes the filesystem or the install. +// series they own. So by default it carries bibliographic facts only: titles, +// contributors, series position, identifiers, runtime, labels. Never by default +// a path, size, codec, format or anything else that describes the filesystem or +// the install: the file is made to be shared, and a path names folders on the +// server (people's names, a share layout) that its owner may not mean to +// publish. Only an admin asking for them (ExportOptions.IncludePaths, for their +// own scripts, which tie a row back to a book) gets each row's library id and +// paths, and the console warns them the file is then theirs alone. // // The envelope is the shape the metadata site's importer already parses; treat // it as a wire contract (additive changes only, `omitempty` everywhere). @@ -55,20 +60,39 @@ type ExportLibrary struct { Name string `json:"name"` } -// ExportBook is one book as the importer sees it: bibliographic facts only. -// Every field but the title is optional and omitted when unknown. (The importer -// also accepts a `subtitle`, which the index has no column for; the envelope is -// additive, so that field appears here the day the index grows one.) +// ExportBook is one book as the importer sees it: bibliographic facts only, +// unless the export was asked for paths. Every field but the title is optional +// and omitted when unknown. The importer reads subtitle, publisher, language and +// abridged and ignores keys it doesn't know (genres, tags, and the paths' two), +// so the envelope's version stays 1. type ExportBook struct { Title string `json:"title"` + Subtitle string `json:"subtitle,omitempty"` Authors []string `json:"authors,omitempty"` Narrators []string `json:"narrators,omitempty"` Series string `json:"series,omitempty"` SeriesPosition string `json:"series_position,omitempty"` + Publisher string `json:"publisher,omitempty"` + Language string `json:"language,omitempty"` + Abridged *bool `json:"abridged,omitempty"` // absent when unknown + Genres []string `json:"genres,omitempty"` + Tags []string `json:"tags,omitempty"` ASIN string `json:"asin,omitempty"` ISBN string `json:"isbn,omitempty"` RuntimeMin int `json:"runtime_min,omitempty"` Chapters int `json:"chapters,omitempty"` + // LibraryID and Paths tie the row back to the books it stands for: the + // library and the path of every copy that collapsed into it (index order). + // Only with ExportOptions.IncludePaths; absent otherwise. + LibraryID int64 `json:"library_id,omitempty"` + Paths []string `json:"paths,omitempty"` +} + +// ExportOptions is what an export holds beyond the bibliographic facts. +type ExportOptions struct { + // IncludePaths adds each row's library id and paths (ExportBook.Paths): for + // the admin's own scripts, never for a file meant to be shared. + IncludePaths bool } // Filename is the file name to offer the download as: @@ -88,7 +112,7 @@ func (e *LibraryExport) Filename() string { // // serverVersion is passed in rather than read from the api package: catalog must // not depend on the transport layer. -func (c *Catalog) ExportLibraryBooks(ctx context.Context, libraryID int64, serverVersion string) (*LibraryExport, error) { +func (c *Catalog) ExportLibraryBooks(ctx context.Context, libraryID int64, serverVersion string, opt ExportOptions) (*LibraryExport, error) { lib, err := c.GetLibrary(ctx, libraryID) if err != nil { return nil, err @@ -127,6 +151,9 @@ func (c *Catalog) ExportLibraryBooks(ctx context.Context, libraryID int64, serve for i := range page.Books { b := &page.Books[i] eb := exportBook(b, counts[b.ID]) + if opt.IncludePaths { + eb.LibraryID, eb.Paths = b.LibraryID, []string{b.RelPath} + } if key := exposedDedupKey(*b); key != "" { if at, ok := seen[key]; ok { mergeExportBook(&out.Books[at], eb) @@ -150,10 +177,16 @@ func (c *Catalog) ExportLibraryBooks(ctx context.Context, libraryID int64, serve func exportBook(b *Book, chapters int) ExportBook { return ExportBook{ Title: strings.TrimSpace(b.Title), + Subtitle: strings.TrimSpace(b.Subtitle), Authors: names.Split(b.Author), Narrators: names.Split(b.Narrator), Series: strings.TrimSpace(b.Series), SeriesPosition: FormatSeriesRange(b.SeriesIndex, b.SeriesIndexEnd), + Publisher: strings.TrimSpace(b.Publisher), + Language: strings.TrimSpace(b.Language), + Abridged: b.Abridged, + Genres: b.Genres, + Tags: b.Tags, ASIN: strings.TrimSpace(b.ASIN), ISBN: strings.TrimSpace(b.ISBN), RuntimeMin: runtimeMinutes(b.Duration), @@ -166,11 +199,32 @@ func exportBook(b *Book, chapters int) ExportBook { // independently, so the one that happens to sort first can be the sparsely-tagged // rip (same asin, but no narrator/series tags) while the fully-tagged copy sorts // later; taking each fact from whichever copy has it keeps the export as complete -// as the library is, without ever overwriting a fact already recorded. +// as the library is, without ever overwriting a fact already recorded. The +// copy's paths are added to the entry's (an export with paths names every copy +// a row stands for). func mergeExportBook(dst *ExportBook, src ExportBook) { + dst.Paths = append(dst.Paths, src.Paths...) if dst.Title == "" { dst.Title = src.Title } + if dst.Subtitle == "" { + dst.Subtitle = src.Subtitle + } + if dst.Publisher == "" { + dst.Publisher = src.Publisher + } + if dst.Language == "" { + dst.Language = src.Language + } + if dst.Abridged == nil { + dst.Abridged = src.Abridged + } + if len(dst.Genres) == 0 { + dst.Genres = src.Genres + } + if len(dst.Tags) == 0 { + dst.Tags = src.Tags + } if len(dst.Authors) == 0 { dst.Authors = src.Authors } diff --git a/internal/catalog/export_test.go b/internal/catalog/export_test.go index ae1382a2..6690421a 100644 --- a/internal/catalog/export_test.go +++ b/internal/catalog/export_test.go @@ -4,6 +4,7 @@ import ( "encoding/json" "errors" "reflect" + "slices" "strconv" "strings" "testing" @@ -119,7 +120,7 @@ func TestExportLibraryBooks(t *testing.T) { t.Fatal(err) } - exp, err := c.ExportLibraryBooks(ctx, lib.ID, "1.2.3") + exp, err := c.ExportLibraryBooks(ctx, lib.ID, "1.2.3", ExportOptions{}) if err != nil { t.Fatalf("ExportLibraryBooks: %v", err) } @@ -200,7 +201,7 @@ func TestExportLibraryBooksMergesDuplicateCopies(t *testing.T) { t.Fatal(err) } - exp, err := c.ExportLibraryBooks(ctx, lib.ID, "dev") + exp, err := c.ExportLibraryBooks(ctx, lib.ID, "dev", ExportOptions{}) if err != nil { t.Fatal(err) } @@ -230,7 +231,7 @@ func TestExportLibraryBooksLeaksNoFilesystem(t *testing.T) { }); err != nil { t.Fatal(err) } - exp, err := c.ExportLibraryBooks(ctx, lib.ID, "dev") + exp, err := c.ExportLibraryBooks(ctx, lib.ID, "dev", ExportOptions{}) if err != nil { t.Fatal(err) } @@ -283,7 +284,69 @@ func TestExportLibraryBooksLeaksNoFilesystem(t *testing.T) { t.Fatalf("books = %d, want 1: %s", len(books), raw) } assertKeys("book", books[0], "title", "subtitle", "authors", "narrators", - "series", "series_position", "asin", "isbn", "runtime_min", "chapters") + "series", "series_position", "publisher", "language", "abridged", "genres", "tags", + "asin", "isbn", "runtime_min", "chapters") +} + +// TestExportLibraryBooksIncludePaths: asked for paths, each row names its library +// and the path of every copy collapsed into it, and the leak guard's allowlist +// holds otherwise; without the option neither key appears (above). +func TestExportLibraryBooksIncludePaths(t *testing.T) { + t.Parallel() + c, ctx := newTestCatalog(t) + lib, _ := c.CreateLibrary(ctx, Library{Name: "Main", Root: "/srv/secret-root"}) + for _, b := range []*Book{ + {LibraryID: lib.ID, RelPath: "Rips/Die Trying.mp3", Title: "Die Trying", Author: "Lee Child", ASIN: "B0TESTASIN", Size: 10}, + {LibraryID: lib.ID, RelPath: "Lee Child/Die Trying", IsFolder: true, Title: "Die Trying", Author: "Lee Child", + ASIN: "B0TESTASIN", Genres: []string{"Thriller"}, Size: 100}, + {LibraryID: lib.ID, RelPath: "Other/Book.m4b", Title: "Other", Author: "Someone", Size: 5}, + } { + if _, err := c.UpsertBook(ctx, b); err != nil { + t.Fatal(err) + } + } + exp, err := c.ExportLibraryBooks(ctx, lib.ID, "dev", ExportOptions{IncludePaths: true}) + if err != nil { + t.Fatal(err) + } + if len(exp.Books) != 2 { + t.Fatalf("books = %d, want 2: %+v", len(exp.Books), exp.Books) + } + die, other := exp.Books[0], exp.Books[1] + if die.LibraryID != lib.ID || !slices.Equal(die.Paths, []string{"Rips/Die Trying.mp3", "Lee Child/Die Trying"}) || + !slices.Equal(die.Genres, []string{"Thriller"}) { + t.Errorf("collapsed row = %+v, want both copies' paths and the genre", die) + } + if other.LibraryID != lib.ID || !slices.Equal(other.Paths, []string{"Other/Book.m4b"}) { + t.Errorf("single row = %+v", other) + } + raw, _ := json.Marshal(exp) + var doc struct { + Library map[string]json.RawMessage `json:"library"` + Books []map[string]json.RawMessage `json:"books"` + } + if err := json.Unmarshal(raw, &doc); err != nil { + t.Fatal(err) + } + if _, ok := doc.Library["root"]; ok || strings.Contains(string(raw), "secret-root") { + t.Errorf("the library root leaked: %s", raw) + } + allowed := map[string]bool{"title": true, "authors": true, "asin": true, "genres": true, "library_id": true, "paths": true} + for _, b := range doc.Books { + for k := range b { + if !allowed[k] { + t.Errorf("row carries unexpected field %q: %s", k, raw) + } + } + } + + // The same library without the option: no row names a path or a library. + exp, _ = c.ExportLibraryBooks(ctx, lib.ID, "dev", ExportOptions{}) + raw, _ = json.Marshal(exp) + if strings.Contains(string(raw), `"paths"`) || strings.Contains(string(raw), `"library_id"`) || + strings.Contains(string(raw), "Rips/") { + t.Errorf("an export without paths carries them: %s", raw) + } } // TestExportLibraryBooksPagesEveryBook checks the keyset paging loop drains a @@ -301,7 +364,7 @@ func TestExportLibraryBooksPagesEveryBook(t *testing.T) { t.Fatal(err) } } - exp, err := c.ExportLibraryBooks(ctx, lib.ID, "dev") + exp, err := c.ExportLibraryBooks(ctx, lib.ID, "dev", ExportOptions{}) if err != nil { t.Fatal(err) } @@ -320,7 +383,7 @@ func TestExportLibraryBooksPagesEveryBook(t *testing.T) { func TestExportLibraryBooksUnknownLibrary(t *testing.T) { t.Parallel() c, ctx := newTestCatalog(t) - if _, err := c.ExportLibraryBooks(ctx, 999, "dev"); !errors.Is(err, ErrNotFound) { + if _, err := c.ExportLibraryBooks(ctx, 999, "dev", ExportOptions{}); !errors.Is(err, ErrNotFound) { t.Fatalf("err = %v, want ErrNotFound", err) } } diff --git a/internal/catalog/labels.go b/internal/catalog/labels.go new file mode 100644 index 00000000..53897f7b --- /dev/null +++ b/internal/catalog/labels.go @@ -0,0 +1,298 @@ +package catalog + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "maps" + "slices" + "strings" +) + +// A book's genres and tags are labels: a list of short names each, with no order +// that means anything beyond the one they were given in. Genres come from the +// files (the genre tag, metadata.SplitGenres) and an Audiobookshelf +// metadata.json; tags only from that file and an admin's edit (no audio tag +// carries them). Both are override fields like any other - provenance, revert, +// undo and a bulk edit work on them unchanged - whose value is the list as +// canonical JSON ("" for none), as more_series is. A bulk edit can also add +// labels to each book's own list or take some away (BookEdit.Add, Remove), since +// setting one list on many books would wipe what each had. +// +// A label is one label however it is cased ("Sci-Fi" and "sci-fi" are one tag), +// spacing aside: refreshEffective writes each book's labels to book_labels by +// that key (labelKey), which the genre= and tag= filters and the facet counts +// read by its index. Derived, like book_more_series: rebuilt with the books row. + +// The label fields, and the kinds book_labels files them under. +const ( + FieldGenres = "genres" + FieldTags = "tags" + + labelGenre = "genre" + labelTag = "tag" +) + +// labelKinds maps each label field to its book_labels kind. +var labelKinds = map[string]string{FieldGenres: labelGenre, FieldTags: labelTag} + +// Bounds on a label list: a real book has a handful, and a label is a word or +// a short phrase. +const ( + maxLabelLen = 64 + maxLabels = 50 +) + +// labelKey is the key a label is one label by: its words, lowercased. +func labelKey(name string) string { return strings.ToLower(collapseSpace(name)) } + +// collapseSpace trims s and joins its words with one space. +func collapseSpace(s string) string { return strings.Join(strings.Fields(s), " ") } + +// cleanLabels is in as a label list: each label trimmed with its inner spacing +// collapsed, blanks dropped, each label once (labelKey: the first spelling +// kept). Strict (an edit) refuses a label over maxLabelLen, one with a control +// character and a list over maxLabels, naming field; lenient (what a file says) +// leaves such a label out and keeps the first maxLabels, so one odd value never +// costs a book the rest. Nil for none. +func cleanLabels(field string, in []string, strict bool) ([]string, error) { + var out []string + seen := map[string]bool{} + for _, raw := range in { + name := collapseSpace(raw) + if name == "" || seen[labelKey(name)] { + continue + } + if err := checkText(field, name, maxLabelLen); err != nil { + if strict { + return nil, invalid(field, "a label must be at most 64 characters, with no control characters") + } + continue + } + seen[labelKey(name)] = true + out = append(out, name) + } + if len(out) > maxLabels { + if strict { + return nil, invalid(field, "too many labels (at most 50)") + } + out = out[:maxLabels] + } + return out, nil +} + +// CleanLabels is a label list a file gives, as a label field stores it (the +// lenient cleanLabels): what the scan and an ABS file record. +func CleanLabels(in []string) []string { + out, _ := cleanLabels("", in, false) + return out +} + +// normalizeLabels validates a label field's value (a JSON list of strings) and +// returns its canonical form ("" for an empty list). +func normalizeLabels(field, v string) (string, error) { + if v == "" { + return "", nil + } + var in []string + if err := json.Unmarshal([]byte(v), &in); err != nil { + return "", invalid(field, "must be a list of names") + } + out, err := cleanLabels(field, in, true) + if err != nil { + return "", err + } + return EncodeLabels(out), nil +} + +// EncodeLabels is the canonical value of a label list ("" for none): without +// HTML escaping, so "Growing Up & Facts of Life" is written as the console's +// JSON.stringify writes it and the two compare equal. +func EncodeLabels(labels []string) string { + if len(labels) == 0 { + return "" + } + var b strings.Builder + enc := json.NewEncoder(&b) + enc.SetEscapeHTML(false) + _ = enc.Encode(labels) // a list of strings always encodes + return strings.TrimSuffix(b.String(), "\n") +} + +// ParseLabels reads a stored label value (canonical JSON; "" for none). A value +// it can't read names no label. +func ParseLabels(v string) []string { + if v == "" || v == "[]" { + return nil + } + var out []string + _ = json.Unmarshal([]byte(v), &out) + return out +} + +// mergeLabels is the stored list cur with the labels of add that it lacks +// appended and those remove names (by labelKey) taken out: how a bulk edit +// changes each book's own list. add is already clean (cleanLabels) and names +// none of remove (BookEdit.normalize); a list that would pass maxLabels is +// refused. +func mergeLabels(field, cur string, add, remove []string) (string, error) { + drop := map[string]bool{} + for _, r := range remove { + drop[labelKey(r)] = true + } + out := slices.DeleteFunc(ParseLabels(cur), func(l string) bool { return drop[labelKey(l)] }) + for _, a := range add { + if !slices.ContainsFunc(out, func(l string) bool { return labelKey(l) == labelKey(a) }) { + out = append(out, a) + } + } + if len(out) > maxLabels { + return "", invalid(field, "too many labels (at most 50)") + } + return EncodeLabels(out), nil +} + +// normalizeLabelEdits checks a BookEdit's Add or Remove (which, what) and +// returns it clean: only the label fields take one, never a field the edit also +// sets or reverts; added labels as an edit stores them, removed ones by name +// (blank ones dropped). +func normalizeLabelEdits(e BookEdit, which string, in map[string][]string) (map[string][]string, error) { + out := make(map[string][]string, len(in)) + for field, labels := range in { + if _, ok := labelKinds[field]; !ok { + return nil, invalid(field, "only genres and tags take "+which) + } + if touches(e, field) { + return nil, invalid(field, "cannot be set or reverted and changed by "+which+" at once") + } + var clean []string + if which == "add" { + var err error + if clean, err = cleanLabels(field, labels, true); err != nil { + return nil, err + } + } else { + for _, l := range labels { + if l = collapseSpace(l); l != "" { + clean = append(clean, l) + } + } + } + out[field] = clean + } + return out, nil +} + +// labelSets is what an edit's Add and Remove make of one book's lists: the new +// value of each list field it changes (the book's effective list, books.genres / +// tags, with the labels added and taken out), leaving out a list it leaves as it +// was, so adding a tag a book already has edits nothing on it. +func labelSets(ctx context.Context, tx *sql.Tx, bookID int64, edit BookEdit) (map[string]string, error) { + if len(edit.Add) == 0 && len(edit.Remove) == 0 { + return nil, nil + } + var genres, tags string + if err := tx.QueryRowContext(ctx, `SELECT genres, tags FROM books WHERE id = ?`, bookID).Scan(&genres, &tags); err != nil { + return nil, err + } + cur := map[string]string{FieldGenres: genres, FieldTags: tags} + out := map[string]string{} + for _, field := range slices.Sorted(maps.Keys(labelKinds)) { + add, remove := edit.Add[field], edit.Remove[field] + if len(add) == 0 && len(remove) == 0 { + continue + } + v, err := mergeLabels(field, cur[field], add, remove) + if err != nil { + return nil, err + } + if v != cur[field] { + out[field] = v + } + } + return out, nil +} + +// writeLabels writes book_labels for a book: each of its genres and tags by key, +// with the spelling the book uses. +func writeLabels(ctx context.Context, tx *sql.Tx, bookID, libID int64, genres, tags string) error { + if _, err := tx.ExecContext(ctx, `DELETE FROM book_labels WHERE book_id = ?`, bookID); err != nil { + return err + } + for kind, v := range map[string]string{labelGenre: genres, labelTag: tags} { + for _, name := range ParseLabels(v) { + if _, err := tx.ExecContext(ctx, + `INSERT OR IGNORE INTO book_labels(book_id, library_id, kind, label_key, name) VALUES(?, ?, ?, ?, ?)`, + bookID, libID, kind, labelKey(name), name); err != nil { + return err + } + } + } + return nil +} + +// labelFilter is the WHERE condition, and its args, for a genre= or tag= filter +// (kind) on the books row whose columns carry prefix: the books that have the +// label named name, in any case (labelKey), found by book_labels' index. +func labelFilter(prefix, kind, name string) (string, []any) { + return prefix + "id IN (SELECT bl.book_id FROM book_labels bl WHERE bl.kind = ? AND bl.label_key = ?)", + []any{kind, labelKey(name)} +} + +// ftsLabels is the full-text index's text for a label list: a label a line. +func ftsLabels(v string) string { return strings.Join(ParseLabels(v), "\n") } + +// SetGenres records, by path, the genres a read of each book's primary file +// found (the scanner's backfill for rows indexed before migration 0049, which +// never had their genre tag read), marking each checked: the genres go into the +// book's scanned snapshot, as an index would have put them, and the book is +// re-resolved when that changes its snapshot. It commits every pathReadingBatch +// books. +func (c *Catalog) SetGenres(ctx context.Context, libraryID int64, genres map[string][]string) error { + return c.eachPathBatch(ctx, "SetGenres", slices.Collect(maps.Keys(genres)), + func(tx *sql.Tx, relPath string) (bool, error) { + return setGenres(ctx, tx, libraryID, relPath, genres[relPath]) + }) +} + +func setGenres(ctx context.Context, tx *sql.Tx, libraryID int64, relPath string, genres []string) (bool, error) { + id, err := bookIDByPath(ctx, tx, libraryID, relPath) + if errors.Is(err, ErrNotFound) { + return false, nil + } + if err != nil { + return false, err + } + if _, err := tx.ExecContext(ctx, `UPDATE books SET genres_checked = 1 WHERE id = ?`, id); err != nil { + return false, err + } + l, err := loadLayers(ctx, tx, id) + if err != nil { + return false, err + } + v := EncodeLabels(CleanLabels(genres)) + if l.scanned[FieldGenres] == v { + return false, nil // most books: no genre, or only a placeholder + } + if l.fromRow != "" { + // The snapshot is read off an older server's row, which wrote its chapters + // with no scanned_title: keep their titles, as setPathReading does. + if _, err := tx.ExecContext(ctx, `UPDATE chapters SET scanned_title = title WHERE book_id = ?`, id); err != nil { + return false, err + } + } + snap := maps.Clone(l.scanned) + snap[FieldGenres] = v + if l.pathRev != "" { + snap[scannedRevKey] = l.pathRev + } + raw, err := encodeScanned(snap, l.indexedAt) + if err != nil { + return false, err + } + if _, err := tx.ExecContext(ctx, `UPDATE books SET scanned = ? WHERE id = ?`, raw, id); err != nil { + return false, err + } + return true, refreshEffective(ctx, tx, id) +} diff --git a/internal/catalog/labels_test.go b/internal/catalog/labels_test.go new file mode 100644 index 00000000..430e55e9 --- /dev/null +++ b/internal/catalog/labels_test.go @@ -0,0 +1,387 @@ +package catalog + +import ( + "encoding/json" + "errors" + "reflect" + "slices" + "strconv" + "strings" + "testing" + + "github.com/kodestar/audiosilo-server/internal/metadata" +) + +func TestNormalizeLabels(t *testing.T) { + t.Parallel() + for _, tc := range []struct{ in, want string }{ + {"", ""}, + {"[]", ""}, + {`[" Sci-Fi ", "Space Opera", "sci-fi", "", "SPACE OPERA", "Tom & Jerry"]`, `["Sci-Fi","Space Opera","Tom & Jerry"]`}, + } { + got, err := normalizeLabels(FieldTags, tc.in) + if err != nil || got != tc.want { + t.Errorf("normalizeLabels(%s) = %q, %v; want %q", tc.in, got, err, tc.want) + } + } + many := make([]string, maxLabels+1) + for i := range many { + many[i] = "t" + strconv.Itoa(i) + } + raw, _ := json.Marshal(many) + for name, in := range map[string]string{ + "not a list": `"Fantasy"`, + "not strings": `[1, 2]`, + "too long": `["` + strings.Repeat("x", maxLabelLen+1) + `"]`, + "control": `["a\u0007b"]`, + "too many": string(raw), + "numbers mixed": `["a", 3]`, + } { + if _, err := normalizeLabels(FieldGenres, in); !errors.Is(err, ErrInvalidOverride) { + t.Errorf("%s: err = %v, want an invalid override", name, err) + } + } + // A label of exactly the limit is fine. + if _, err := normalizeLabels(FieldTags, `["`+strings.Repeat("é", maxLabelLen)+`"]`); err != nil { + t.Errorf("a %d-character label: %v", maxLabelLen, err) + } +} + +// What a file gives is cleaned, never refused: an odd label is left out and a +// long list cut, so the rest of the book's labels stay. +func TestCleanLabels(t *testing.T) { + t.Parallel() + in := []string{"Fantasy", strings.Repeat("x", maxLabelLen+1), "a\x07b", " fantasy ", "Epic"} + if got := CleanLabels(in); !reflect.DeepEqual(got, []string{"Fantasy", "Epic"}) { + t.Errorf("CleanLabels = %q", got) + } + many := make([]string, maxLabels+5) + for i := range many { + many[i] = "t" + strconv.Itoa(i) + } + if got := CleanLabels(many); len(got) != maxLabels || got[0] != "t0" { + t.Errorf("CleanLabels kept %d labels, want the first %d", len(got), maxLabels) + } + if CleanLabels(nil) != nil { + t.Error("no labels should be nil") + } +} + +func TestMergeLabels(t *testing.T) { + t.Parallel() + got, err := mergeLabels(FieldTags, `["Owned","Read"]`, []string{"read", "Favourite"}, []string{" OWNED "}) + if err != nil || got != `["Read","Favourite"]` { + t.Errorf("merge = %q, %v", got, err) + } + if got, _ := mergeLabels(FieldTags, `["Owned"]`, nil, []string{"owned"}); got != "" { + t.Errorf("removing the last label = %q, want none", got) + } + full := make([]string, maxLabels) + for i := range full { + full[i] = "t" + strconv.Itoa(i) + } + if _, err := mergeLabels(FieldTags, EncodeLabels(full), []string{"one more"}, nil); !errors.Is(err, ErrInvalidOverride) { + t.Errorf("past the cap: err = %v, want an invalid override", err) + } +} + +// TestNormalizeNewTextFields: subtitle, publisher and language are trimmed text; +// abridged is true, false or unknown. +func TestNormalizeNewTextFields(t *testing.T) { + t.Parallel() + for _, tc := range []struct{ field, in, want string }{ + {FieldSubtitle, " A Novel ", "A Novel"}, + {FieldPublisher, "Tantor Audio", "Tantor Audio"}, + {FieldLanguage, " en-GB ", "en-GB"}, + {FieldAbridged, "TRUE", "true"}, + {FieldAbridged, "false", "false"}, + {FieldAbridged, "", ""}, + } { + if got, err := normalizeOverride(tc.field, tc.in); err != nil || got != tc.want { + t.Errorf("%s %q = %q, %v; want %q", tc.field, tc.in, got, err, tc.want) + } + } + for field, in := range map[string]string{ + FieldAbridged: "yes", FieldLanguage: strings.Repeat("x", maxLanguage+1), FieldSubtitle: "a\nb", + } { + if _, err := normalizeOverride(field, in); !errors.Is(err, ErrInvalidOverride) { + t.Errorf("%s %q: err = %v, want an invalid override", field, in, err) + } + } +} + +// TestLabelLayers: genres from the tags, the ABS file's over them (with its +// tags, subtitle, publisher, language and abridged), an edit over both, and an +// edit kept through a rescan that brings other genres. The effective values +// reach the player's book (list and single reads alike). +func TestLabelLayers(t *testing.T) { + t.Parallel() + c, ctx := newTestCatalog(t) + lib, _ := c.CreateLibrary(ctx, Library{Name: "L", Root: "/tmp/l"}) + const p = "Author/Book" + scan := func(genres []string, abs *metadata.ABSFile) { + t.Helper() + if _, err := c.UpsertBook(ctx, &Book{ + LibraryID: lib.ID, RelPath: p, IsFolder: true, Title: "Book", Author: "Author", Genres: genres, + ABS: ABSReading{Sig: "1:1", File: abs}, + }); err != nil { + t.Fatal(err) + } + } + scan([]string{"Fantasy", "Epic"}, nil) + d, err := c.AdminBookDetail(ctx, lib.ID, p) + if err != nil { + t.Fatal(err) + } + if got := d.Fields[FieldGenres]; got.Value != `["Fantasy","Epic"]` || got.Source != SourceTag { + t.Errorf("scanned genres = %+v", got) + } + if got := d.Book.Genres; !reflect.DeepEqual(got, []string{"Fantasy", "Epic"}) { + t.Errorf("admin row genres = %q", got) + } + + scan([]string{"Fantasy"}, &metadata.ABSFile{ + Subtitle: "A Tale", Publisher: "Tantor", Language: "English", Abridged: true, + Genres: []string{"Horror"}, Tags: []string{"Owned", "owned", "To Read"}, + }) + d, _ = c.AdminBookDetail(ctx, lib.ID, p) + for field, want := range map[string]FieldValue{ + FieldSubtitle: {Value: "A Tale", Source: SourceABS}, + FieldPublisher: {Value: "Tantor", Source: SourceABS}, + FieldLanguage: {Value: "English", Source: SourceABS}, + FieldAbridged: {Value: "true", Source: SourceABS}, + FieldGenres: {Value: `["Horror"]`, Source: SourceABS}, + FieldTags: {Value: `["Owned","To Read"]`, Source: SourceABS}, + } { + if got := d.Fields[field]; got.Value != want.Value || got.Source != want.Source { + t.Errorf("%s = %q (%s), want %q (%s)", field, got.Value, got.Source, want.Value, want.Source) + } + } + b, err := c.GetBookByPath(ctx, lib.ID, p) + if err != nil { + t.Fatal(err) + } + if b.Subtitle != "A Tale" || b.Publisher != "Tantor" || b.Language != "English" || b.Abridged == nil || !*b.Abridged || + !slices.Equal(b.Genres, []string{"Horror"}) || !slices.Equal(b.Tags, []string{"Owned", "To Read"}) { + t.Errorf("player book = %+v", b) + } + + // An edit goes over the file, and a rescan with other genres keeps it. + if err := c.EditBook(ctx, lib.ID, p, BookEdit{Set: map[string]string{ + FieldTags: `["Mine"]`, FieldAbridged: "false", + }}); err != nil { + t.Fatal(err) + } + scan([]string{"Mystery"}, &metadata.ABSFile{Tags: []string{"Theirs"}}) + b, _ = c.GetBookByPath(ctx, lib.ID, p) + if !slices.Equal(b.Tags, []string{"Mine"}) || b.Abridged == nil || *b.Abridged || !slices.Equal(b.Genres, []string{"Mystery"}) { + t.Errorf("after a rescan: tags %q, abridged %v, genres %q; want the edits kept and the new genre", b.Tags, b.Abridged, b.Genres) + } + + // Nothing says abridged: unknown, omitted on the wire. + if err := c.EditBook(ctx, lib.ID, p, BookEdit{Revert: []string{FieldAbridged}}); err != nil { + t.Fatal(err) + } + page, err := c.ListBooks(ctx, ListOptions{LibraryID: lib.ID}) + if err != nil || len(page.Books) != 1 { + t.Fatalf("list: %v %v", page, err) + } + raw, _ := json.Marshal(page.Books[0]) + if strings.Contains(string(raw), `"abridged"`) || !strings.Contains(string(raw), `"tags":["Mine"]`) || + !strings.Contains(string(raw), `"genres":["Mystery"]`) { + t.Errorf("list book JSON = %s", raw) + } +} + +// labelBooks seeds a library for the filter, search and bulk tests: three books +// with their own tags and genres. +func labelBooks(t *testing.T) (*Catalog, int64) { + t.Helper() + c, ctx := newTestCatalog(t) + lib, _ := c.CreateLibrary(ctx, Library{Name: "L", Root: "/tmp/l"}) + for _, b := range []struct { + path string + genres []string + tags string + }{ + {"A", []string{"Fantasy"}, `["Owned","Favourite"]`}, + {"B", []string{"Science Fiction", "fantasy"}, `["owned"]`}, + {"C", []string{"Horror"}, ""}, + } { + if _, err := c.UpsertBook(ctx, &Book{LibraryID: lib.ID, RelPath: b.path, Title: "Book " + b.path, Genres: b.genres}); err != nil { + t.Fatal(err) + } + if b.tags != "" { + if err := c.EditBook(ctx, lib.ID, b.path, BookEdit{Set: map[string]string{FieldTags: b.tags}}); err != nil { + t.Fatal(err) + } + } + } + return c, lib.ID +} + +func listPaths(t *testing.T, c *Catalog, opt ListOptions) []string { + t.Helper() + page, err := c.ListBooks(t.Context(), opt) + if err != nil { + t.Fatal(err) + } + var out []string + for _, b := range page.Books { + out = append(out, b.RelPath) + } + slices.Sort(out) + return out +} + +// TestLabelFilters: genre= and tag= find a book by its label in any case, on the +// player's list and the admin list alike, and a search finds a book by a tag. +func TestLabelFilters(t *testing.T) { + t.Parallel() + c, lib := labelBooks(t) + for name, tc := range map[string]struct { + opt ListOptions + want []string + }{ + "genre": {ListOptions{Genre: "FANTASY"}, []string{"A", "B"}}, + "tag": {ListOptions{Tag: "owned"}, []string{"A", "B"}}, + "both": {ListOptions{Genre: "fantasy", Tag: "favourite"}, []string{"A"}}, + "spaced": {ListOptions{Genre: "science fiction"}, []string{"B"}}, + "none": {ListOptions{Tag: "nobody's"}, nil}, + "genre not tag": {ListOptions{Tag: "Horror"}, nil}, + "tag with scope": {ListOptions{Tag: "owned", Scope: &Scope{LibraryID: lib, Paths: []string{"B"}}}, []string{"B"}}, + } { + tc.opt.LibraryID = lib + if got := listPaths(t, c, tc.opt); !slices.Equal(got, tc.want) { + t.Errorf("%s: %v, want %v", name, got, tc.want) + } + } + page, err := c.ListAdminBooks(t.Context(), AdminListOptions{Filter: BookFilter{LibraryID: lib, Tag: "OWNED", Genre: "horror"}}) + if err != nil || len(page.Books) != 0 { + t.Errorf("admin owned+horror = %v, %v; want none", page, err) + } + page, _ = c.ListAdminBooks(t.Context(), AdminListOptions{Filter: BookFilter{LibraryID: lib, Tag: "favourite"}}) + if len(page.Books) != 1 || page.Books[0].Path != "A" || !slices.Equal(page.Books[0].Tags, []string{"Owned", "Favourite"}) { + t.Errorf("admin favourite = %+v", page.Books) + } + + found, err := c.Search(t.Context(), "favour", []Scope{{LibraryID: lib, AllowAll: true}}, 10) + if err != nil || len(found) != 1 || found[0].RelPath != "A" { + t.Errorf("search by tag = %v, %v; want A", found, err) + } + found, _ = c.Search(t.Context(), "horror", []Scope{{LibraryID: lib, AllowAll: true}}, 10) + if len(found) != 1 || found[0].RelPath != "C" { + t.Errorf("search by genre = %v; want C", found) + } +} + +// TestLabelFacets: each label counted by key, named by the spelling most of its +// books use, without its own filter. +func TestLabelFacets(t *testing.T) { + t.Parallel() + c, lib := labelBooks(t) + f, err := c.BookFacets(t.Context(), BookFilter{LibraryID: lib, Genre: "horror"}) + if err != nil { + t.Fatal(err) + } + wantGenres := []FacetCount{{"Fantasy", 2}, {"Horror", 1}, {"Science Fiction", 1}} + if !reflect.DeepEqual(f.Genres, wantGenres) { + t.Errorf("genres = %+v, want %+v", f.Genres, wantGenres) + } + if len(f.Tags) != 0 { + t.Errorf("tags of the horror book = %+v, want none", f.Tags) + } + f, _ = c.BookFacets(t.Context(), BookFilter{LibraryID: lib}) + if want := []FacetCount{{"Owned", 2}, {"Favourite", 1}}; !reflect.DeepEqual(f.Tags, want) { + t.Errorf("tags = %+v, want %+v", f.Tags, want) + } +} + +// TestBulkLabelEdit: adding and removing labels keeps each book's own others, +// writes nothing on a book it leaves as it was, and refuses to be mixed with a +// set or a revert of the same field. +func TestBulkLabelEdit(t *testing.T) { + t.Parallel() + c, lib := labelBooks(t) + ctx := t.Context() + refs := []Ref{{LibraryID: lib, Path: "A"}, {LibraryID: lib, Path: "B"}, {LibraryID: lib, Path: "C"}} + if err := c.EditBooks(ctx, refs, BookEdit{ + Add: map[string][]string{FieldTags: {"Read", " read "}}, + Remove: map[string][]string{FieldTags: {"OWNED"}, FieldGenres: {"horror"}}, + }); err != nil { + t.Fatal(err) + } + want := map[string][2][]string{ // tags, genres + "A": {{"Favourite", "Read"}, {"Fantasy"}}, + "B": {{"Read"}, {"Science Fiction", "fantasy"}}, + "C": {{"Read"}, nil}, + } + for path, w := range want { + b, _ := c.GetBookByPath(ctx, lib, path) + if !slices.Equal(b.Tags, w[0]) || !slices.Equal(b.Genres, w[1]) { + t.Errorf("%s: tags %q genres %q, want %q %q", path, b.Tags, b.Genres, w[0], w[1]) + } + } + // C's genre removal is an edit (the scan's Horror stays off through a rescan); + // A's genres were untouched, so they stay unlocked. + d, _ := c.AdminBookDetail(ctx, lib, "A") + if d.Fields[FieldGenres].Locked { + t.Error("A's genres were locked by a removal that changed nothing on it") + } + if _, err := c.UpsertBook(ctx, &Book{LibraryID: lib, RelPath: "C", Title: "Book C", Genres: []string{"Horror"}}); err != nil { + t.Fatal(err) + } + if b, _ := c.GetBookByPath(ctx, lib, "C"); b.Genres != nil { + t.Errorf("C after a rescan: genres %q, want the removal kept", b.Genres) + } + + for name, edit := range map[string]BookEdit{ + "set and add": {Set: map[string]string{FieldTags: `["x"]`}, Add: map[string][]string{FieldTags: {"y"}}}, + "revert and remove": {Revert: []string{FieldGenres}, Remove: map[string][]string{FieldGenres: {"y"}}}, + "add and remove": {Add: map[string][]string{FieldTags: {"Same"}}, Remove: map[string][]string{FieldTags: {"same"}}}, + "not a label": {Add: map[string][]string{FieldTitle: {"y"}}}, + "too long": {Add: map[string][]string{FieldTags: {strings.Repeat("x", maxLabelLen+1)}}}, + } { + if err := c.EditBooks(ctx, refs, edit); !errors.Is(err, ErrInvalidOverride) { + t.Errorf("%s: err = %v, want an invalid override", name, err) + } + } +} + +// TestSetGenres: the backfill of a book indexed before genres were read puts +// them in its snapshot (source tag, an edit still over them) and marks it +// checked, so Signatures stops asking. +func TestSetGenres(t *testing.T) { + t.Parallel() + c, ctx := newTestCatalog(t) + lib, _ := c.CreateLibrary(ctx, Library{Name: "L", Root: "/tmp/l"}) + for _, p := range []string{"Old", "None"} { + if _, err := c.UpsertBook(ctx, &Book{LibraryID: lib.ID, RelPath: p, Title: p}); err != nil { + t.Fatal(err) + } + } + if _, err := c.db.ExecContext(ctx, `UPDATE books SET genres_checked = 0`); err != nil { + t.Fatal(err) + } + sigs, _ := c.Signatures(ctx, lib.ID) + if !sigs["Old"].GenresUnchecked { + t.Fatal("a book before 0049 should be unchecked") + } + if err := c.SetGenres(ctx, lib.ID, map[string][]string{"Old": {"Fantasy", "fantasy"}, "None": nil}); err != nil { + t.Fatal(err) + } + d, _ := c.AdminBookDetail(ctx, lib.ID, "Old") + if got := d.Fields[FieldGenres]; got.Value != `["Fantasy"]` || got.Source != SourceTag { + t.Errorf("backfilled genres = %+v", got) + } + if got := d.Fields[FieldTitle]; got.Value != "Old" { + t.Errorf("title after the backfill = %+v", got) + } + sigs, _ = c.Signatures(ctx, lib.ID) + if sigs["Old"].GenresUnchecked || sigs["None"].GenresUnchecked { + t.Error("checked books should no longer be unchecked") + } + if got := listPaths(t, c, ListOptions{LibraryID: lib.ID, Genre: "fantasy"}); !slices.Equal(got, []string{"Old"}) { + t.Errorf("genre filter after the backfill = %v", got) + } +} diff --git a/internal/catalog/model.go b/internal/catalog/model.go index 6afebb58..c6c096de 100644 --- a/internal/catalog/model.go +++ b/internal/catalog/model.go @@ -102,6 +102,18 @@ type Book struct { // loads it, leaving list, search and recent pages small. Published string `json:"published,omitempty"` Description string `json:"description,omitempty"` + // Subtitle, Publisher and Language come from an Audiobookshelf metadata.json, + // an edit or a community match. Abridged likewise: nil (omitted) when nothing + // says, so a client never reads "unknown" as "unabridged". Genres come from + // the genre tag too (metadata.SplitGenres), Tags only from the ABS file or an + // edit (see labels.go); both ride on every player book, list pages included, + // so a client can filter a library it has loaded by them. + Subtitle string `json:"subtitle,omitempty"` + Publisher string `json:"publisher,omitempty"` + Language string `json:"language,omitempty"` + Abridged *bool `json:"abridged,omitempty"` + Genres []string `json:"genres,omitempty"` + Tags []string `json:"tags,omitempty"` // Released is the date the file's tags give (metadata.ReleaseDate), usually // the recording's: scanned, never edited, and only the admin list's // release-date sort reads it (where a book has no Published). diff --git a/internal/catalog/overrides.go b/internal/catalog/overrides.go index aaad2c2b..00b60da9 100644 --- a/internal/catalog/overrides.go +++ b/internal/catalog/overrides.go @@ -34,11 +34,16 @@ import ( // Overridable book fields (the wire names the console uses). const ( FieldTitle = "title" + FieldSubtitle = "subtitle" FieldAuthor = "author" FieldNarrator = "narrator" FieldSeries = "series" FieldSeriesIndex = "series_index" FieldPublished = "published" + FieldPublisher = "publisher" + FieldLanguage = "language" + // FieldAbridged is "true", "false", or "" when unknown (books.abridged NULL). + FieldAbridged = "abridged" FieldDescription = "description" FieldASIN = "asin" FieldISBN = "isbn" @@ -46,8 +51,9 @@ const ( // OverrideFields lists every overridable field, in display order. var OverrideFields = []string{ - FieldTitle, FieldAuthor, FieldNarrator, FieldSeries, FieldSeriesIndex, FieldMoreSeries, - FieldPublished, FieldDescription, FieldASIN, FieldISBN, + FieldTitle, FieldSubtitle, FieldAuthor, FieldNarrator, FieldSeries, FieldSeriesIndex, FieldMoreSeries, + FieldPublished, FieldPublisher, FieldLanguage, FieldAbridged, FieldGenres, FieldTags, + FieldDescription, FieldASIN, FieldISBN, } // Where a field's value came from. A scanned value is SourcePath when it is what @@ -88,6 +94,7 @@ func invalid(field, reason string) error { return &OverrideError{Field: field, R // to second-guess real titles. const ( maxShortField = 500 + maxLanguage = 64 maxDescription = 20000 maxSeriesIndex = metadata.MaxSeriesPosition maxChapterTitle = 500 @@ -111,6 +118,21 @@ func normalizeOverride(field, value string) (string, error) { return "", invalid(field, "a book needs a title") } return v, nil + case FieldSubtitle, FieldPublisher: + return v, checkText(field, v, maxShortField) + case FieldLanguage: + if utf8.RuneCountInString(v) > maxLanguage || strings.IndexFunc(v, unicode.IsControl) >= 0 { + return "", invalid(field, "must be a language name or code, at most 64 characters") + } + return v, nil + case FieldAbridged: + switch v = strings.ToLower(v); v { + case "", "true", "false": + return v, nil + } + return "", invalid(field, `must be "true", "false" or empty`) + case FieldGenres, FieldTags: + return normalizeLabels(field, v) case FieldDescription: // A description keeps its line breaks (normalized to \n); any other // control character is refused. @@ -185,16 +207,43 @@ func validDatePrefix(v string) bool { // range). type bookFields map[string]string -// fieldsOf reads a book's overridable fields. +// fieldsOf reads a book's overridable fields. A label list (genres, tags) is +// cleaned as a file's is (CleanLabels): this is what a scan records. func fieldsOf(b *Book) bookFields { return bookFields{ - FieldTitle: b.Title, FieldAuthor: b.Author, FieldNarrator: b.Narrator, + FieldTitle: b.Title, FieldSubtitle: b.Subtitle, FieldAuthor: b.Author, FieldNarrator: b.Narrator, FieldSeries: b.Series, FieldSeriesIndex: FormatSeriesRange(b.SeriesIndex, b.SeriesIndexEnd), - FieldPublished: b.Published, FieldDescription: b.Description, + FieldPublished: b.Published, FieldPublisher: b.Publisher, FieldLanguage: b.Language, + FieldAbridged: formatAbridged(b.Abridged), FieldGenres: EncodeLabels(CleanLabels(b.Genres)), + FieldTags: EncodeLabels(CleanLabels(b.Tags)), FieldDescription: b.Description, FieldASIN: b.ASIN, FieldISBN: b.ISBN, } } +// formatAbridged is an abridged flag in its field form: "true", "false", or "" +// when unknown (nil). +func formatAbridged(b *bool) string { + switch { + case b == nil: + return "" + case *b: + return "true" + } + return "false" +} + +// abridgedColumn is an abridged field value as books.abridged stores it: 1, 0, +// or NULL when unknown. +func abridgedColumn(v string) any { + switch v { + case "true": + return 1 + case "false": + return 0 + } + return nil +} + // ParseSeriesIndex reads a stored series_index value as the position a client // knows: the number, or a range's start ("1-3" is 1); 0 for none. func ParseSeriesIndex(s string) float64 { @@ -572,7 +621,8 @@ func pathFields(m *metadata.Metadata) bookFields { } // refreshEffective rebuilds a book's effective metadata (bookLayers.resolve) into -// its books row (with its series keys: series_key, book_more_series), its chapters (their source: applyChapterSource; then their +// its books row (with its series keys: series_key, book_more_series; and its +// labels: book_labels), its chapters (their source: applyChapterSource; then their // titles), then refreshes its FTS row. UpsertBook, // SetEnrichment and every edit call it, so the books row can never disagree with // what the durable tables say. @@ -601,16 +651,21 @@ func refreshEffective(ctx context.Context, tx *sql.Tx, bookID int64) error { v := func(field string) string { return fields[field].Value } start, end := ParseSeriesPosition(v(FieldSeriesIndex)) if _, err := tx.ExecContext(ctx, - `UPDATE books SET title = ?, author = ?, narrator = ?, series = ?, series_key = ?, series_index = ?, - series_index_end = ?, more_series = ?, published = ?, description = ?, asin = ?, isbn = ? + `UPDATE books SET title = ?, subtitle = ?, author = ?, narrator = ?, series = ?, series_key = ?, series_index = ?, + series_index_end = ?, more_series = ?, published = ?, publisher = ?, language = ?, abridged = ?, + genres = ?, tags = ?, description = ?, asin = ?, isbn = ? WHERE id = ?`, - v(FieldTitle), v(FieldAuthor), v(FieldNarrator), v(FieldSeries), seriesKey(v(FieldSeries)), start, - end, cmp.Or(v(FieldMoreSeries), "[]"), v(FieldPublished), v(FieldDescription), v(FieldASIN), v(FieldISBN), bookID); err != nil { + v(FieldTitle), v(FieldSubtitle), v(FieldAuthor), v(FieldNarrator), v(FieldSeries), seriesKey(v(FieldSeries)), start, + end, cmp.Or(v(FieldMoreSeries), "[]"), v(FieldPublished), v(FieldPublisher), v(FieldLanguage), abridgedColumn(v(FieldAbridged)), + v(FieldGenres), v(FieldTags), v(FieldDescription), v(FieldASIN), v(FieldISBN), bookID); err != nil { return err } if err := keyOtherSeries(ctx, tx, bookID, l.libID, v(FieldSeries), v(FieldMoreSeries)); err != nil { return err } + if err := writeLabels(ctx, tx, bookID, l.libID, v(FieldGenres), v(FieldTags)); err != nil { + return err + } if err := applyChapterSource(ctx, tx, bookID, l.libID, l.path); err != nil { return err @@ -630,13 +685,16 @@ func refreshEffective(ctx context.Context, tx *sql.Tx, bookID int64) error { // Refresh FTS: delete-then-insert keyed by rowid = book id, from the effective // values just written. The series column holds every series the book is in, - // so a search finds it by any (the series= filter goes by key instead). + // so a search finds it by any (the series= filter goes by key instead), and + // the genres and tags columns a label a line, so a search finds a book by its + // labels too (the genre= and tag= filters go by book_labels). if _, err := tx.ExecContext(ctx, `DELETE FROM books_fts WHERE rowid = ?`, bookID); err != nil { return err } _, err = tx.ExecContext(ctx, - `INSERT INTO books_fts(rowid, title, author, series, narrator) VALUES(?,?,?,?,?)`, - bookID, v(FieldTitle), v(FieldAuthor), ftsSeries(v(FieldSeries), v(FieldMoreSeries)), v(FieldNarrator)) + `INSERT INTO books_fts(rowid, title, author, series, narrator, genres, tags) VALUES(?,?,?,?,?,?,?)`, + bookID, v(FieldTitle), v(FieldAuthor), ftsSeries(v(FieldSeries), v(FieldMoreSeries)), v(FieldNarrator), + ftsLabels(v(FieldGenres)), ftsLabels(v(FieldTags))) return err } @@ -664,8 +722,15 @@ func queryOverrides(ctx context.Context, q rowQuerier, libID int64, path string) // revert to what the scan found, and the same for chapter titles (by index). // Source says where the set values came from (SourceEdited or SourceCommunity). type BookEdit struct { - Set map[string]string - Revert []string + Set map[string]string + Revert []string + // Add and Remove change a label list (genres, tags) rather than set it: each + // book's own effective list gains the labels Add names that it lacks and + // loses those Remove names (any case), written as an override of the field + // when that changes it (labelSets). A field is never both set or reverted and + // added to or removed from. + Add map[string][]string + Remove map[string][]string ChapterSet map[int]string ChapterRevert []int // ChapterSource picks where the chapters come from: ChaptersFromFiles, @@ -732,7 +797,22 @@ func (e BookEdit) normalize() (BookEdit, error) { default: return e, invalid("chapter_source", `must be "auto", "files" or "community"`) } - e.Set, e.ChapterSet = set, chSet + add, err := normalizeLabelEdits(e, "add", e.Add) + if err != nil { + return e, err + } + remove, err := normalizeLabelEdits(e, "remove", e.Remove) + if err != nil { + return e, err + } + for field, names := range remove { + for _, name := range names { + if slices.ContainsFunc(add[field], func(a string) bool { return labelKey(a) == labelKey(name) }) { + return e, invalid(field, "a label cannot be added and removed at once") + } + } + } + e.Set, e.ChapterSet, e.Add, e.Remove = set, chSet, add, remove return e, nil } @@ -800,12 +880,17 @@ func (c *Catalog) editTx(ctx context.Context, tx *sql.Tx, ref Ref, edit BookEdit if err != nil { return err } + labels, err := labelSets(ctx, tx, bookID, edit) + if err != nil { + return err + } // This book's own copies: EditBooks shares the edit across books. The swap - // writes only fields the edit leaves alone, each either set or reverted, so - // no field is both. - if len(swapSet) > 0 { + // and the label changes write only fields the edit leaves alone, each either + // set or reverted, so no field is both. + if len(swapSet) > 0 || len(labels) > 0 { set = maps.Clone(edit.Set) maps.Copy(set, swapSet) + maps.Copy(set, labels) } if len(swapRevert) > 0 { revert = slices.Concat(edit.Revert, swapRevert) diff --git a/internal/catalog/seriesrange_test.go b/internal/catalog/seriesrange_test.go index 28f189dd..504f4b09 100644 --- a/internal/catalog/seriesrange_test.go +++ b/internal/catalog/seriesrange_test.go @@ -136,7 +136,7 @@ func TestSeriesRangeEditAndRevert(t *testing.T) { if err != nil || len(series) != 1 || !reflect.DeepEqual(series[0].Positions, []float64{1, 2, 3, 4, 5, 6}) { t.Errorf("series = %+v, %v; want positions 1 to 6", series, err) } - exp, err := c.ExportLibraryBooks(ctx, lib.ID, "dev") + exp, err := c.ExportLibraryBooks(ctx, lib.ID, "dev", ExportOptions{}) if err != nil { t.Fatal(err) } diff --git a/internal/library/labels_test.go b/internal/library/labels_test.go new file mode 100644 index 00000000..209dceca --- /dev/null +++ b/internal/library/labels_test.go @@ -0,0 +1,94 @@ +package library + +import ( + "log/slog" + "os" + "path/filepath" + "slices" + "testing" + "time" + + "github.com/kodestar/audiosilo-server/internal/catalog" + "github.com/kodestar/audiosilo-server/internal/store/storetest" +) + +// A scan reads a book's genre tag (placeholders dropped, several in one tag +// split) and an Audiobookshelf metadata.json's subtitle, publisher, language, +// abridged, genres and tags; a book indexed before migration 0049 gets both on +// the next scan, its genre tag read once and its ABS file again, without a +// re-index, and an edit made meanwhile is kept. +func TestScanReadsLabels(t *testing.T) { + t.Parallel() + ctx := t.Context() + db := storetest.Open(t) + cat := catalog.New(db, time.Now) + root := t.TempDir() + silence(t, root, "Herbert/Dune/01.mp3", 1, "genre=Audiobook;Science Fiction") + silence(t, root, "Pratchett/Mort/01.mp3", 1, "genre=Fantasy") + if err := os.WriteFile(filepath.Join(root, "Pratchett", "Mort", "metadata.json"), []byte(`{ + "subtitle": "A Discworld Novel", "publisher": "Corgi", "language": "English", "abridged": true, + "genres": ["Fantasy", "Humour"], "tags": ["Owned", "Favourite"] + }`), 0o600); err != nil { + t.Fatal(err) + } + lib, _ := cat.CreateLibrary(ctx, catalog.Library{Name: "L", Root: root}) + s := NewScanner(cat, "", slog.Default()) + if _, err := s.Scan(ctx, *lib); err != nil { + t.Fatal(err) + } + check := func(when string) { + t.Helper() + dune, err := cat.GetBookByPath(ctx, lib.ID, "Herbert/Dune") + if err != nil { + t.Fatal(err) + } + if !slices.Equal(dune.Genres, []string{"Science Fiction"}) { + t.Errorf("%s: Dune genres = %q, want [Science Fiction]", when, dune.Genres) + } + mort, err := cat.GetBookByPath(ctx, lib.ID, "Pratchett/Mort") + if err != nil { + t.Fatal(err) + } + if mort.Subtitle != "A Discworld Novel" || mort.Publisher != "Corgi" || mort.Language != "English" || + mort.Abridged == nil || !*mort.Abridged || !slices.Equal(mort.Genres, []string{"Fantasy", "Humour"}) || + !slices.Equal(mort.Tags, []string{"Mine"}) { + t.Errorf("%s: Mort = subtitle %q publisher %q language %q abridged %v genres %q tags %q", when, + mort.Subtitle, mort.Publisher, mort.Language, mort.Abridged, mort.Genres, mort.Tags) + } + } + if err := cat.EditBook(ctx, lib.ID, "Pratchett/Mort", + catalog.BookEdit{Set: map[string]string{catalog.FieldTags: `["Mine"]`}}); err != nil { + t.Fatal(err) + } + check("indexed") + + // Rows from before 0049: no genre read, an ABS reading without the new fields + // (and an effective row without them). The next scan puts them back without + // re-indexing the (unchanged) books. + if _, err := db.ExecContext(ctx, `UPDATE books SET genres_checked = 0, abs_indexed_at = NULL, + abs_file = CASE WHEN abs_file = '' THEN '' ELSE + json_remove(abs_file, '$.subtitle', '$.publisher', '$.language', '$.abridged', '$.genres', '$.tags') END, + scanned = json_remove(scanned, '$.genres'), subtitle = '', publisher = '', language = '', abridged = NULL, + genres = '', tags = ''`); err != nil { + t.Fatal(err) + } + if _, err := db.ExecContext(ctx, `DELETE FROM book_labels`); err != nil { + t.Fatal(err) + } + res, err := s.Scan(ctx, *lib) + if err != nil || res.Updated != 0 { + t.Fatalf("rescan = %+v (err %v), want the labels backfilled, nothing re-indexed", res.ScanCounts, err) + } + check("backfilled") + var unchecked int + if err := db.QueryRowContext(ctx, `SELECT COUNT(*) FROM books WHERE genres_checked = 0`).Scan(&unchecked); err != nil || unchecked != 0 { + t.Fatalf("unchecked after the backfill = %d (err %v), want 0", unchecked, err) + } + page, err := cat.ListBooks(ctx, catalog.ListOptions{LibraryID: lib.ID, Genre: "science fiction"}) + if err != nil || len(page.Books) != 1 || page.Books[0].RelPath != "Herbert/Dune" { + t.Fatalf("genre filter after the backfill = %+v (err %v)", page, err) + } + if res, err := s.Scan(ctx, *lib); err != nil || res.Updated != 0 { + t.Fatalf("third scan = %+v (err %v), want nothing re-indexed once checked", res.ScanCounts, err) + } +} diff --git a/internal/library/scanner.go b/internal/library/scanner.go index 610b0ab7..c10bb050 100644 --- a/internal/library/scanner.go +++ b/internal/library/scanner.go @@ -266,6 +266,10 @@ func (s *Scanner) Scan(ctx context.Context, lib catalog.Library) (_ *ScanResult, suspectBackfill := map[string]int{} splitBackfill := map[string]string{} releasedBackfill := map[string]string{} + // Books indexed before 0049 whose genre tag was never read: the genres a read + // of the primary file finds (SetGenres), flushed like the path readings since + // each one that has any re-resolves its book. + genresBackfill := map[string][]string{} pathBackfill := map[string]*metadata.Metadata{} // Folder books whose metadata.json is new, changed, gone or never read, while // their audio is unchanged: read here and recorded without a re-index @@ -274,7 +278,7 @@ func (s *Scanner) Scan(ctx context.Context, lib catalog.Library) (_ *ScanResult, // Books whose .audiosilo/ attachments are new, changed, gone or never read, // while their audio is unchanged: likewise (SetAttachments). attachBackfill := map[string]catalog.AttachmentsReading{} - // The path, ABS and attachments backfills are written as they go: a large one + // The path, genre, ABS and attachments backfills are written as they go: a large one // (most of a library laid out Author/Title, or every folder book of a library // ABS has managed, read once after an upgrade) can take a while, and a scan // stopped part way keeps what it read (written past the Stop: at most @@ -284,6 +288,10 @@ func (s *Scanner) Scan(ctx context.Context, lib catalog.Library) (_ *ScanResult, s.log.Warn("record path readings failed", "library", lib.Name, "err", err) } clear(pathBackfill) + if err := s.cat.SetGenres(context.WithoutCancel(ctx), lib.ID, genresBackfill); err != nil { + s.log.Warn("record genres failed", "library", lib.Name, "err", err) + } + clear(genresBackfill) if err := s.cat.SetABSFiles(context.WithoutCancel(ctx), lib.ID, absBackfill); err != nil { s.log.Warn("record Audiobookshelf metadata failed", "library", lib.Name, "err", err) } @@ -340,21 +348,25 @@ func (s *Scanner) Scan(ctx context.Context, lib catalog.Library) (_ *ScanResult, suspectBackfill[b.RelPath] = n } } - // Rows indexed before 0037 haven't had their date read, and some books - // one folder deep, read when that folder was taken for their series, - // need their tags to tell (catalog.PathRead): one read of the primary - // file (tags and ffprobe, as enrich reads it) does both, not a re-index - // probing every part. Unopenable now, it waits for the next scan; and + // Rows indexed before 0037 haven't had their date read, nor rows + // before 0049 their genre, and some books one folder deep, read when + // that folder was taken for their series, need their tags to tell + // (catalog.PathRead): one read of the primary file (tags and ffprobe, + // as enrich reads it) does all three, not a re-index probing every + // part. Unopenable now, it waits for the next scan; and // the names wait while ffprobe fails where it didn't before, as the // series it alone reads (an MP3's TXXX) would be recorded as none. if old.PathCheck == catalog.PathFromSnapshot { pathBackfill[b.RelPath] = nil } - if old.ReleasedUnchecked || old.PathCheck == catalog.PathRead { + if old.ReleasedUnchecked || old.GenresUnchecked || old.PathCheck == catalog.PathRead { if md, _ := metadata.Extract(primary, s.ffprobePath); md.OpenErr == nil { if old.ReleasedUnchecked { releasedBackfill[b.RelPath] = md.Released } + if old.GenresUnchecked { + genresBackfill[b.RelPath] = md.Genres + } if old.PathCheck == catalog.PathRead && (md.ProbeErr == nil || old.ScanError == problemProbe) { pathBackfill[b.RelPath] = readNames(b.RelPath, b.IsFolder, md) } @@ -379,7 +391,7 @@ func (s *Scanner) Scan(ctx context.Context, lib catalog.Library) (_ *ScanResult, Was: &catalog.AttachmentsState{Sig: old.AttachmentsSig, Unchecked: old.AttachmentsUnchecked}} } } - if len(pathBackfill)+len(absBackfill)+len(attachBackfill) >= pathFlushEvery { + if len(pathBackfill)+len(genresBackfill)+len(absBackfill)+len(attachBackfill) >= pathFlushEvery { flushBackfills() } continue @@ -575,6 +587,7 @@ func (s *Scanner) enrich(lib catalog.Library, b *catalog.Book) { if md != nil { b.Narrator = md.Narrator b.Released = md.Released + b.Genres = md.Genres b.Codec = md.Codec } diff --git a/internal/matchrun/plan.go b/internal/matchrun/plan.go index a5f9147c..e889004b 100644 --- a/internal/matchrun/plan.go +++ b/internal/matchrun/plan.go @@ -86,7 +86,11 @@ var yearRE = regexp.MustCompile(`^\d{4}`) // recording's best for this server (meta.MatchRecording.ASINs is ordered by the // preferred marketplace), named with its marketplace. The series are the ones a // book that is rec is in (meta.MatchRecording.Series): an edition named for that -// recording first, never an edition of another recording. +// recording first, never an edition of another recording. The subtitle and +// language are the work's, the publisher the recording's, and abridged is "true" +// only for a recording marked so: the community's false is also its "not +// recorded", so it never claims a book unabridged. The community's genres are +// not offered (a match candidate carries none). func Propose(c *meta.MatchCandidate, rec *meta.MatchRecording) catalog.MatchProposal { p := catalog.MatchProposal{ WorkID: c.WorkID, Title: c.Title, Authors: names(c.Authors), WebURL: c.WebURL, @@ -94,7 +98,8 @@ func Propose(c *meta.MatchCandidate, rec *meta.MatchRecording) catalog.MatchProp } published := c.FirstPublished raw := map[string]string{ - catalog.FieldTitle: c.Title, catalog.FieldAuthor: p.Authors, catalog.FieldDescription: c.Description, + catalog.FieldTitle: c.Title, catalog.FieldSubtitle: c.Subtitle, catalog.FieldAuthor: p.Authors, + catalog.FieldLanguage: c.Language, catalog.FieldDescription: c.Description, } series := c.Series if rec != nil && rec.Series != nil { @@ -113,7 +118,10 @@ func Propose(c *meta.MatchCandidate, rec *meta.MatchRecording) catalog.MatchProp } if rec != nil { p.RecordingID, p.Narrators, p.RuntimeMin = rec.ID, names(rec.Narrators), rec.RuntimeMin - raw[catalog.FieldNarrator] = p.Narrators + raw[catalog.FieldNarrator], raw[catalog.FieldPublisher] = p.Narrators, rec.Publisher + if rec.Abridged { + raw[catalog.FieldAbridged] = "true" + } if published == "" { published = rec.ReleaseDate } diff --git a/internal/matchrun/plan_test.go b/internal/matchrun/plan_test.go index e5fe53e3..35c1ab61 100644 --- a/internal/matchrun/plan_test.go +++ b/internal/matchrun/plan_test.go @@ -56,6 +56,44 @@ func TestPropose(t *testing.T) { } } +// TestProposeEditionFields: the work's subtitle and language and the recording's +// publisher are offered; abridged only when the recording is marked so (the +// community's false is also "not recorded"). Fill fills them on a book without +// them, overwrite replaces a file's, and an admin's own stays either way. +func TestProposeEditionFields(t *testing.T) { + t.Parallel() + c := martian + c.Subtitle, c.Language = "A Novel", "en" + c.Recordings = slices.Clone(martian.Recordings) + c.Recordings[0].Publisher = "Podium Publishing" + p := Propose(&c, &c.Recordings[0]) + for field, want := range map[string]string{"subtitle": "A Novel", "language": "en", "publisher": "Podium Publishing"} { + if p.Values[field] != want { + t.Errorf("%s = %q, want %q", field, p.Values[field], want) + } + } + if _, ok := p.Values["abridged"]; ok { + t.Errorf("an unmarked recording offered abridged: %v", p.Values) + } + c.Recordings[0].Abridged = true + if p = Propose(&c, &c.Recordings[0]); p.Values["abridged"] != "true" { + t.Errorf("abridged = %q, want true", p.Values["abridged"]) + } + if p := Propose(&c, nil); p.Values["publisher"] != "" || p.Values["subtitle"] != "A Novel" { + t.Errorf("no recording: %v, want the work's subtitle and no publisher", p.Values) + } + + st := state(map[string]string{"title": "The Martian", "language": "English", "publisher": "Mine"}, []string{"publisher"}, false) + set, _ := planSet(ScopeFill, st, p) + if set["subtitle"] != "A Novel" || set["abridged"] != "true" || set["language"] != "" || set["publisher"] != "" { + t.Errorf("fill = %v, want the subtitle and abridged only", set) + } + set, _ = planSet(ScopeOverwrite, st, p) + if set["language"] != "en" || set["publisher"] != "" { + t.Errorf("overwrite = %v, want the language replaced and the admin's publisher kept", set) + } +} + // TestProposeSeriesForTheRecording: the series proposed are the chosen // recording's (an edition named for it first, as the community service's // candidate lists them per recording); with no recording, the work's own. diff --git a/internal/store/migration_0049_test.go b/internal/store/migration_0049_test.go new file mode 100644 index 00000000..583c77c5 --- /dev/null +++ b/internal/store/migration_0049_test.go @@ -0,0 +1,61 @@ +package store + +import ( + "context" + "testing" +) + +// TestMigration0049BookLabels: the books gain the new columns empty (abridged +// unknown), every book is marked for its genre tag's read and every ABS file +// for a read again, and the full-text index made again with the label columns +// keeps every row it had, still searchable. +func TestMigration0049BookLabels(t *testing.T) { + t.Parallel() + ctx := context.Background() + dsn, exec, closeRaw := openBefore(t, "0049") + exec(`INSERT INTO libraries(id, name, root, created_at) VALUES(1, 'Books', '/l', 't')`) + exec(`INSERT INTO books(id, library_id, rel_path, is_folder, title, author, series, narrator, indexed_at, abs_indexed_at, released_checked) + VALUES(7, 1, 'Pratchett/Mort', 1, 'Mort', 'Terry Pratchett', 'Discworld', 'Nigel Planer', 'x', 'x', 1)`) + exec(`INSERT INTO books_fts(rowid, title, author, series, narrator) VALUES(7, 'Mort', 'Terry Pratchett', 'Discworld +Death', 'Nigel Planer')`) + closeRaw() + + db, err := Open(ctx, dsn) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { db.Close() }) + var ( + subtitle, publisher, language, genres, tags string + abridged *int + checked int + absAt *string + ) + if err := db.writer.QueryRow(`SELECT subtitle, publisher, language, abridged, genres, tags, genres_checked, abs_indexed_at + FROM books WHERE id = 7`).Scan(&subtitle, &publisher, &language, &abridged, &genres, &tags, &checked, &absAt); err != nil { + t.Fatal(err) + } + if subtitle != "" || publisher != "" || language != "" || abridged != nil || genres != "" || tags != "" { + t.Errorf("new columns = %q %q %q %v %q %q, want empty and unknown", subtitle, publisher, language, abridged, genres, tags) + } + if checked != 0 || absAt != nil { + t.Errorf("genres_checked = %d, abs_indexed_at = %v; want 0 and NULL (read again)", checked, absAt) + } + var rowid int64 + var series, labelCols string + if err := db.writer.QueryRow(`SELECT rowid, series, genres || tags FROM books_fts WHERE books_fts MATCH 'death'`). + Scan(&rowid, &series, &labelCols); err != nil { + t.Fatal(err) + } + if rowid != 7 || series != "Discworld\nDeath" || labelCols != "" { + t.Errorf("fts row = %d %q %q, want the copied row", rowid, series, labelCols) + } + if _, err := db.writer.Exec(`INSERT INTO books_fts(rowid, title, author, series, narrator, genres, tags) + VALUES(8, 'Guards', '', '', '', 'Fantasy', 'Owned')`); err != nil { + t.Fatal(err) + } + var n int + if err := db.writer.QueryRow(`SELECT COUNT(*) FROM books_fts WHERE books_fts MATCH 'tags: owned'`).Scan(&n); err != nil || n != 1 { + t.Errorf("a tag match = %d, %v; want 1", n, err) + } +} diff --git a/internal/store/migrations/0049_book_labels.sql b/internal/store/migrations/0049_book_labels.sql new file mode 100644 index 00000000..d8f0d8c6 --- /dev/null +++ b/internal/store/migrations/0049_book_labels.sql @@ -0,0 +1,54 @@ +-- More of a book's metadata as effective values (catalog: overrides.go, +-- labels.go), layered like the rest (bookLayers.resolve: the scan, an +-- Audiobookshelf metadata.json, then a community match or an edit): +-- subtitle, publisher, language: text ('' for none), from the ABS file, a +-- match or an edit; +-- abridged: 1 or 0, NULL while nothing says (ABS's false says nothing, nor +-- does a match's); +-- genres, tags: the book's labels, a canonical JSON list ('' for none). +-- Genres come from the genre tag too, tags only from the ABS file or an +-- edit. +-- Effective values like title, rewritten from the layers by refreshEffective. +ALTER TABLE books ADD COLUMN subtitle TEXT NOT NULL DEFAULT ''; +ALTER TABLE books ADD COLUMN publisher TEXT NOT NULL DEFAULT ''; +ALTER TABLE books ADD COLUMN language TEXT NOT NULL DEFAULT ''; +ALTER TABLE books ADD COLUMN abridged INTEGER; +ALTER TABLE books ADD COLUMN genres TEXT NOT NULL DEFAULT ''; +ALTER TABLE books ADD COLUMN tags TEXT NOT NULL DEFAULT ''; + +-- A book indexed before this never had its genre tag read, so it starts +-- unchecked (0) and the next scan reads its primary file once (tags and +-- ffprobe, no re-index; catalog.SetGenres), like 0037's released_checked; every +-- index from now on writes 1. A rebuildable index flag. +ALTER TABLE books ADD COLUMN genres_checked INTEGER NOT NULL DEFAULT 0; + +-- Every metadata.json was read before its subtitle, publisher, language, +-- abridged, genres and tags were: marking each folder book's file unread for +-- this index (0046's abs_indexed_at) has the next scan read it again, without a +-- re-index. +UPDATE books SET abs_indexed_at = NULL; + +-- Each book's labels by key (catalog.labelKey: lowercased, spacing collapsed), +-- the genre= and tag= filters' and the facet counts' index, with the spelling +-- the book uses. Written by Go (refreshEffective, beside book_more_series) from +-- the effective lists, never computed in SQL. Derived, rebuildable index data +-- like the books rows they belong to; every row before this has no labels, so +-- the table starts empty. +CREATE TABLE book_labels ( + book_id INTEGER NOT NULL REFERENCES books(id) ON DELETE CASCADE, + library_id INTEGER NOT NULL, + kind TEXT NOT NULL CHECK (kind IN ('genre', 'tag')), + label_key TEXT NOT NULL, + name TEXT NOT NULL, + PRIMARY KEY (book_id, kind, label_key) +) WITHOUT ROWID; +CREATE INDEX idx_book_labels_key ON book_labels(kind, label_key, library_id); + +-- The full-text index gains the labels (a label a line), so a search finds a +-- book by a genre or a tag. FTS5 can't add a column, so it is made again with +-- every row copied over: no book has a label yet, so the copy is the whole index. +CREATE VIRTUAL TABLE books_fts_new USING fts5(title, author, series, narrator, genres, tags); +INSERT INTO books_fts_new(rowid, title, author, series, narrator, genres, tags) + SELECT rowid, title, author, series, narrator, '', '' FROM books_fts; +DROP TABLE books_fts; +ALTER TABLE books_fts_new RENAME TO books_fts; From 19f0a4ef9634705e6d03c601a6afcd4014c3e178 Mon Sep 17 00:00:00 2001 From: KodeStar Date: Fri, 16 Oct 2026 11:24:46 +0100 Subject: [PATCH 3/5] Console: genres and tags as chips, subtitle, publisher, language and abridged on the book page; bulk add/remove of labels, genre and tag facets, and the export with file paths --- admin-ui/STYLEGUIDE.md | 5 +- admin-ui/src/api/client.ts | 12 +- admin-ui/src/api/types.ts | 27 ++++ admin-ui/src/components/label-input.tsx | 142 ++++++++++++++++++ admin-ui/src/features/book/book-model.test.ts | 26 ++++ admin-ui/src/features/book/book-model.ts | 36 ++++- admin-ui/src/features/book/book.test.tsx | 31 ++++ admin-ui/src/features/book/details-card.tsx | 101 ++++++++++++- admin-ui/src/features/book/match-dialog.tsx | 6 +- .../src/features/book/match-model.test.ts | 27 +++- admin-ui/src/features/book/match-model.ts | 20 ++- admin-ui/src/features/book/save-changes.tsx | 6 +- .../src/features/libraries/libraries.test.tsx | 34 +++++ .../src/features/libraries/library-card.tsx | 31 +++- .../library/books/books-model.test.ts | 21 +++ .../src/features/library/books/books-model.ts | 43 +++++- .../src/features/library/books/books.test.tsx | 65 ++++++++ .../library/books/bulk-edit-dialog.tsx | 89 +++++++++-- .../features/library/books/filter-labels.ts | 4 + .../features/library/books/filter-sheet.tsx | 21 ++- .../features/library/library-search.test.ts | 4 + .../src/features/library/library-search.ts | 5 + admin-ui/src/i18n/locales/de.json | 33 ++++ admin-ui/src/i18n/locales/en.json | 33 ++++ admin-ui/src/i18n/locales/es.json | 33 ++++ admin-ui/src/i18n/locales/fr.json | 33 ++++ admin-ui/src/i18n/locales/it.json | 33 ++++ admin-ui/src/i18n/locales/pt.json | 33 ++++ admin-ui/src/lib/labels.test.ts | 34 +++++ admin-ui/src/lib/labels.ts | 84 +++++++++++ admin-ui/src/test/library-fixtures.ts | 17 +++ 31 files changed, 1049 insertions(+), 40 deletions(-) create mode 100644 admin-ui/src/components/label-input.tsx create mode 100644 admin-ui/src/lib/labels.test.ts create mode 100644 admin-ui/src/lib/labels.ts diff --git a/admin-ui/STYLEGUIDE.md b/admin-ui/STYLEGUIDE.md index 7fb16ac5..82f34656 100644 --- a/admin-ui/STYLEGUIDE.md +++ b/admin-ui/STYLEGUIDE.md @@ -505,7 +505,10 @@ before → Edited). dashed pink outline. Field row = label (130px) · click-to-edit value · marker (+ "Revert" for edited fields, whose tooltip shows the file-tag value). Editing locks the field against rescans; the legend under the Details card explains the five sources (Audiobookshelf: a -`metadata.json` ABS keeps in the book's folder, over the tags). +`metadata.json` ABS keeps in the book's folder, over the tags). A label list (genres, tags) shows as +secondary 24px chips and edits as a chip box (`LabelInput`: Enter or a comma adds, a chip's x removes, +Backspace takes the last back; the bulk edit's add/remove boxes are the same component); abridged is a +three-way select (not known, abridged, unabridged) that drafts on change. ### Chapter ribbon _custom_ diff --git a/admin-ui/src/api/client.ts b/admin-ui/src/api/client.ts index ccd83511..bd5e6969 100644 --- a/admin-ui/src/api/client.ts +++ b/admin-ui/src/api/client.ts @@ -246,6 +246,9 @@ export interface BookFilter { author?: string; series?: string; narrator?: string; + /** A genre or a tag the book has, in any case. */ + genre?: string; + tag?: string; format?: string[]; codec?: string[]; direct_playable?: boolean; @@ -674,10 +677,13 @@ export const api = { /** * Downloads a library's book list (GET /admin/libraries/{id}/export) as the file - * the server names. Returns the file name. + * the server names. With `paths`, each row also names its library and the paths + * of the copies it stands for (?include=paths): a file for the admin's own + * scripts, not one to share. Returns the file name. */ -export function downloadLibraryExport(id: number): Promise { - return download(`/admin/libraries/${id}/export`, `audiosilo-library-${id}.json`); +export function downloadLibraryExport(id: number, opts: { paths?: boolean } = {}): Promise { + const query = opts.paths ? '?include=paths' : ''; + return download(`/admin/libraries/${id}/export${query}`, `audiosilo-library-${id}.json`); } /** Downloads a backup (GET /admin/backups/{name}). Returns the file name. */ diff --git a/admin-ui/src/api/types.ts b/admin-ui/src/api/types.ts index a837ea52..6569e17f 100644 --- a/admin-ui/src/api/types.ts +++ b/admin-ui/src/api/types.ts @@ -1345,6 +1345,7 @@ export interface AdminBook { path: string; is_folder: boolean; title: string; + subtitle: string; author: string; narrator: string; /** @@ -1362,6 +1363,13 @@ export interface AdminBook { /** Every series the book is in, its main one (`series`) first, with its position in each. */ series_list: SeriesRef[]; published: string; + publisher: string; + language: string; + /** null while nothing says (catalog.AdminBook.Abridged). */ + abridged: boolean | null; + /** The book's labels (catalog labels.go), [] for none. */ + genres: string[]; + tags: string[]; /** Seconds. */ duration: number; format: string; @@ -1437,6 +1445,9 @@ export interface BookFacets { has_chapters: BoolFacet; matched: BoolFacet; edited: BoolFacet; + /** The most common labels (at most 100 each), named by the spelling most of their books use. */ + genres: FacetCount[]; + tags: FacetCount[]; } /** @@ -1510,18 +1521,28 @@ export interface SeriesRef { /** The overridable book fields (catalog.OverrideFields), in display order. */ export const OVERRIDE_FIELDS = [ 'title', + 'subtitle', 'author', 'narrator', 'series', 'series_index', 'more_series', 'published', + 'publisher', + 'language', + 'abridged', + 'genres', + 'tags', 'description', 'asin', 'isbn', ] as const; export type OverrideField = (typeof OVERRIDE_FIELDS)[number]; +/** The label fields (catalog labels.go): a list as canonical JSON, "" for none. */ +export const LABEL_FIELDS = ['genres', 'tags'] as const; +export type LabelField = (typeof LABEL_FIELDS)[number]; + /** Where a field's value came from (catalog.Source*), in the legend's order. */ export const FIELD_SOURCES = ['tag', 'path', 'abs', 'edited', 'community'] as const; /** A field's source; "" = no value. */ @@ -1671,6 +1692,12 @@ export interface CommunityChaptersDetail { export interface BookEditRequest { set?: Partial>; revert?: OverrideField[]; + /** + * Labels to add to, or take out of (any case), each book's own list; never a + * field the request also sets or reverts (catalog.BookEdit.Add). + */ + add?: Partial>; + remove?: Partial>; /** "community" when the values were accepted from a match; default "edited". */ source?: 'edited' | 'community'; chapters?: { set?: Record; revert?: number[] }; diff --git a/admin-ui/src/components/label-input.tsx b/admin-ui/src/components/label-input.tsx new file mode 100644 index 00000000..d0715330 --- /dev/null +++ b/admin-ui/src/components/label-input.tsx @@ -0,0 +1,142 @@ +import { useLayoutEffect, useRef, useState } from 'react'; +import { useTranslation } from 'react-i18next'; +import { X } from 'lucide-react'; +import { fieldClasses } from '@/components/ui/field-classes'; +import { collapse, labelKey } from '@/lib/labels'; +import { cn } from '@/lib/utils'; + +/** + * A list of labels (genres, tags) edited as chips: type a label and press Enter + * or a comma to add it (a pasted "a, b" adds both), Backspace in the empty box + * takes the last one back, a chip's x removes it. A label already in the list + * (any case) isn't added twice. Controlled: `value` is the list, `onChange` gets + * the next one. With `onDone` it is an in-place editor (the Details card): + * focused on open, Enter in the empty box or leaving it commits (whatever is + * typed is added first), Escape cancels (`onDone(undefined, true)`), and + * `keyboard` says whether a key ended it, so focus can go back to the value. + */ +export function LabelInput({ + id, + value, + onChange, + onDone, + placeholder, + disabled, + autoFocus, + className, + ...aria +}: { + id?: string; + value: string[]; + onChange: (next: string[]) => void; + onDone?: (value: string[] | undefined, keyboard: boolean) => void; + placeholder?: string; + disabled?: boolean; + autoFocus?: boolean; + className?: string; +} & React.AriaAttributes) { + const { t } = useTranslation(); + const [text, setText] = useState(''); + const input = useRef(null); + const box = useRef(null); + const ended = useRef(false); + + useLayoutEffect(() => { + if (autoFocus) input.current?.focus(); + }, [autoFocus]); + + /** value with the labels typed in `raw` (comma-separated) added. */ + const withTyped = (raw: string): string[] => { + const next = [...value]; + for (const part of raw.split(',')) { + const name = collapse(part); + if (name && !next.some((l) => labelKey(l) === labelKey(name))) next.push(name); + } + return next; + }; + + const addTyped = (raw: string) => { + const next = withTyped(raw); + setText(''); + if (next.length !== value.length) onChange(next); + return next; + }; + + const end = (next: string[] | undefined, keyboard: boolean) => { + if (!onDone || ended.current) return; + ended.current = true; + onDone(next, keyboard); + }; + + const remove = (label: string) => { + onChange(value.filter((l) => l !== label)); + input.current?.focus(); + }; + + return ( +
{ + // Leaving the whole box (not moving to one of its chips' buttons) commits. + if (onDone && !box.current?.contains(e.relatedTarget as Node | null)) { + end(withTyped(text), false); + } + }} + > + {value.map((label) => ( + + {label} + + + ))} + { + const v = e.target.value; + // A comma ends a label (a pasted list adds every label in it). + if (v.includes(',')) addTyped(v); + else setText(v); + }} + onKeyDown={(e) => { + if (e.key === 'Enter') { + e.preventDefault(); + if (text.trim()) addTyped(text); + else end(value, true); + } else if (e.key === 'Escape' && onDone) { + e.preventDefault(); + e.stopPropagation(); + end(undefined, true); + } else if (e.key === 'Backspace' && !text && value.length) { + e.preventDefault(); + setText(value[value.length - 1]); + onChange(value.slice(0, -1)); + } + }} + /> +
+ ); +} diff --git a/admin-ui/src/features/book/book-model.test.ts b/admin-ui/src/features/book/book-model.test.ts index 09bdffed..28a142a4 100644 --- a/admin-ui/src/features/book/book-model.test.ts +++ b/admin-ui/src/features/book/book-model.test.ts @@ -12,10 +12,36 @@ import { fileStrip, ribbonSegments, saveRequest, + shownValue, undoRevertRequest, + displayValue, } from './book-model'; describe('checkField', () => { + it('checks the subtitle, publisher, language and abridged as the server does', () => { + expect(checkField('subtitle', ' A Novel ')).toEqual({ value: 'A Novel' }); + expect(checkField('publisher', 'x'.repeat(501)).error).toBe('book.invalid.tooLong'); + expect(checkField('language', ' en-GB ')).toEqual({ value: 'en-GB' }); + expect(checkField('language', 'x'.repeat(65)).error).toBe('book.invalid.language'); + expect(checkField('abridged', 'TRUE')).toEqual({ value: 'true' }); + expect(checkField('abridged', '')).toEqual({ value: '' }); + expect(checkField('abridged', 'yes').error).toBe('book.invalid.abridged'); + }); + + it('keeps a label list canonical, shown as its labels', () => { + expect(checkField('tags', '["Owned","owned"," To Read "]')).toEqual({ + value: '["Owned","To Read"]', + }); + expect(checkField('genres', '["' + 'x'.repeat(65) + '"]').error).toBe( + 'book.invalid.labelTooLong', + ); + expect(displayValue('genres', '["Fantasy","Epic"]')).toBe('Fantasy, Epic'); + const t = (k: string) => `<${k}>`; + expect(shownValue(t, 'abridged', 'false')).toBe(''); + expect(shownValue(t, 'abridged', '')).toBe(''); + expect(shownValue(t, 'tags', '["Owned"]')).toBe('Owned'); + }); + it('trims, and refuses an empty title', () => { expect(checkField('title', ' Dune ')).toEqual({ value: 'Dune' }); expect(checkField('title', ' ').error).toBe('book.invalid.title'); diff --git a/admin-ui/src/features/book/book-model.ts b/admin-ui/src/features/book/book-model.ts index 3d0c0137..5872f040 100644 --- a/admin-ui/src/features/book/book-model.ts +++ b/admin-ui/src/features/book/book-model.ts @@ -12,6 +12,7 @@ import { type OverrideField, } from '@/api/types'; import { formatSeriesPosition } from '@/lib/format'; +import { checkLabels, labelsText } from '@/lib/labels'; import type { Phrase } from '@/lib/phrase'; import { sameSeries } from '@/lib/series-key'; import { CONTROL, MAX_SERIES_INDEX, MAX_SHORT, NUMBER, parseSeriesRange } from './field-rules'; @@ -38,6 +39,7 @@ export interface FieldCheck { } const MAX_DESCRIPTION = 20000; +const MAX_LANGUAGE = 64; // A description keeps line breaks and tabs. // eslint-disable-next-line no-control-regex const CONTROL_IN_TEXT = /[\u0000-\u0008\u000b-\u001f\u007f-\u009f]/; @@ -62,13 +64,27 @@ export function checkField(field: OverrideField, raw: string): FieldCheck { let v = raw.trim(); switch (field) { case 'title': + case 'subtitle': case 'author': case 'narrator': case 'series': + case 'publisher': if (runes(v) > MAX_SHORT) return { value: v, error: 'book.invalid.tooLong' }; if (CONTROL.test(v)) return { value: v, error: 'book.invalid.control' }; if (field === 'title' && !v) return { value: v, error: 'book.invalid.title' }; return { value: v }; + case 'language': + if (runes(v) > MAX_LANGUAGE || CONTROL.test(v)) { + return { value: v, error: 'book.invalid.language' }; + } + return { value: v }; + case 'abridged': + v = v.toLowerCase(); + if (v && v !== 'true' && v !== 'false') return { value: v, error: 'book.invalid.abridged' }; + return { value: v }; + case 'genres': + case 'tags': + return checkLabels(v); case 'description': v = v.replaceAll('\r', ''); if (runes(v) > MAX_DESCRIPTION) return { value: v, error: 'book.invalid.tooLong' }; @@ -103,9 +119,25 @@ export function checkField(field: OverrideField, raw: string): FieldCheck { } } -/** A stored value as it is shown and edited: more_series as its line, the rest as they are. */ +/** + * A stored value as it is shown and edited: more_series as its line, a label + * list as its labels apart by commas, the rest as they are (abridged's words: + * shownValue). + */ export function displayValue(field: OverrideField, value: string): string { - return field === 'more_series' ? moreSeriesText(value) : value; + if (field === 'more_series') return moreSeriesText(value); + if (field === 'genres' || field === 'tags') return labelsText(value); + return value; +} + +/** A stored value in words: displayValue, with abridged's "true"/"false" said. */ +export function shownValue( + t: (key: string) => string, + field: OverrideField, + value: string, +): string { + if (field === 'abridged') return value ? t(`book.abridged.${value}`) : ''; + return displayValue(field, value); } /** diff --git a/admin-ui/src/features/book/book.test.tsx b/admin-ui/src/features/book/book.test.tsx index 24a3d37c..cd1644f3 100644 --- a/admin-ui/src/features/book/book.test.tsx +++ b/admin-ui/src/features/book/book.test.tsx @@ -267,6 +267,33 @@ describe('book page', () => { expect(screen.getByText('2 unsaved changes')).toBeInTheDocument(); }); + it('edits tags as chips and abridged as a choice, saved as edits', async () => { + const detail = bookDetail(); + detail.fields.tags = { value: '["Owned"]', source: 'abs', scanned: '["Owned"]', locked: false }; + const calls = mockFetch(routes(detail)); + renderApp(URL); + const user = userEvent.setup(); + const tags = await screen.findByRole('group', { name: 'Tags' }); + expect(within(tags).getByText('Owned')).toBeInTheDocument(); + await user.click(within(tags).getByRole('button')); + const box = within(row('Tags')).getByRole('textbox', { name: 'Tags' }); + expect(box).toHaveFocus(); + // A label already there (any case) isn't added twice; a comma ends a label. + await user.type(box, 'owned, Favourite,'); + await user.click(within(row('Tags')).getByRole('button', { name: 'Remove Owned' })); + await user.type(within(row('Tags')).getByRole('textbox'), '{Enter}'); + expect(within(row('Tags')).getByText('Favourite')).toBeInTheDocument(); + expect(within(row('Tags')).queryByText('Owned')).toBeNull(); + + await user.selectOptions(within(row('Abridged')).getByRole('combobox'), 'Unabridged'); + await user.click(screen.getByRole('button', { name: 'Review and save' })); + const save = await screen.findByRole('dialog'); + expect(within(save).getByText('Unabridged')).toBeInTheDocument(); + await user.click(within(save).getByRole('button', { name: 'Save 2 changes' })); + await waitFor(() => expect(patches(calls)).toHaveLength(1)); + expect(patches(calls)[0].body).toEqual({ set: { abridged: 'false', tags: '["Favourite"]' } }); + }); + it('reverts a locked field to the file, with an undo', async () => { const detail = bookDetail(); detail.fields.title = { @@ -511,7 +538,11 @@ describe('match with community metadata', () => { const description = within(compare).getByRole('checkbox', { name: 'Accept Description' }); expect(description).toBeChecked(); expect(within(compare).queryByRole('checkbox', { name: 'Accept Title' })).toBeNull(); + // The recording's publisher is offered too. + const publisher = within(compare).getByRole('checkbox', { name: 'Accept Publisher' }); + expect(publisher).toBeChecked(); await user.click(description); + await user.click(publisher); await user.click(within(compare).getByRole('button', { name: 'Accept 1 field' })); expect(await screen.findByText('Accepted 1 field from the community')).toBeInTheDocument(); diff --git a/admin-ui/src/features/book/details-card.tsx b/admin-ui/src/features/book/details-card.tsx index 38ac37be..9a3d2ca4 100644 --- a/admin-ui/src/features/book/details-card.tsx +++ b/admin-ui/src/features/book/details-card.tsx @@ -12,12 +12,15 @@ import { type OverrideField, } from '@/api/types'; import { InlineEdit } from '@/components/inline-edit'; +import { LabelInput } from '@/components/label-input'; import { ProvenanceMarker, UnsavedMarker } from '@/components/provenance'; import { Card, CardHeader } from '@/components/ui/card'; +import { NativeSelect } from '@/components/ui/native-select'; import { toastError } from '@/lib/errors'; +import { encodeLabels, parseLabels } from '@/lib/labels'; import { toast } from '@/lib/toast'; import { cn } from '@/lib/utils'; -import { displayValue, undoRevertRequest, type Drafts } from './book-model'; +import { displayValue, shownValue, undoRevertRequest, type Drafts } from './book-model'; /** How much of a value a toast quotes. */ const QUOTE = 60; @@ -68,7 +71,7 @@ export function DetailsCard({ try { const next = await api.editBook(b.library_id, b.path, { revert: [field] }); settleBookEdit(qc, next); - const now = displayValue(field, next.fields[field]?.value ?? ''); + const now = shownValue(t, field, next.fields[field]?.value ?? ''); toast.add({ title: t('book.revert.done', { field: label }), description: now @@ -122,7 +125,13 @@ export function DetailsCard({ ); } -/** One field: label, the click-to-edit value, and where the value came from. */ +/** The values the abridged field takes, unknown first. */ +const ABRIDGED = ['', 'true', 'false'] as const; + +/** + * One field: label, the value (click to edit; a label list as chips, edited as + * chips; abridged a choice of three), and where the value came from. + */ function FieldRow({ field, value, @@ -150,6 +159,9 @@ function FieldRow({ const id = `field-${field}`; const long = field === 'description'; const mono = field === 'asin' || field === 'isbn'; + const labels = field === 'genres' || field === 'tags'; + // A label list's draft is its JSON (LabelInput's list), shown as chips. + const chips = labels ? parseLabels(draft ?? value.value) : []; // After Enter or Escape, keyboard focus goes back to the value. useEffect(() => { @@ -180,7 +192,33 @@ function FieldRow({ > {label} - {editing ? ( + {field === 'abridged' ? ( + onCommit(e.target.value)} + aria-labelledby={`${id}-label`} + aria-invalid={!!error || undefined} + aria-describedby={error ? `${id}-error` : undefined} + className="max-w-[240px]" + > + {ABRIDGED.map((v) => ( + + ))} + + ) : editing && labels ? ( + + finish(next === undefined ? undefined : encodeLabels(next), keyboard) + } + /> + ) : editing ? ( - - {empty ? t('book.field.empty') : shown} - + {labels && chips.length ? ( + + {chips.map((c) => ( + + {c} + + ))} + + ) : ( + + {empty ? t('book.field.empty') : shown} + + )} )} @@ -224,7 +275,7 @@ function FieldRow({ onClick={onRevert} title={ value.scanned - ? t('book.revert.tagHint', { value: quote(displayValue(field, value.scanned)) }) + ? t('book.revert.tagHint', { value: quote(shownValue(t, field, value.scanned)) }) : t('book.revert.noTag') } aria-label={t('book.revert.aria', { field: label })} @@ -247,3 +298,37 @@ function FieldRow({ ); } + +/** + * A label list being edited in place: the chips and the box to add to them, + * committed as a whole when the editor ends (LabelInput's onDone). + */ +function LabelsEdit({ + id, + initial, + label, + error, + onDone, +}: { + id: string; + initial: string[]; + label: string; + error: string | undefined; + onDone: (value: string[] | undefined, keyboard: boolean) => void; +}) { + const { t } = useTranslation(); + const [list, setList] = useState(initial); + return ( + + ); +} diff --git a/admin-ui/src/features/book/match-dialog.tsx b/admin-ui/src/features/book/match-dialog.tsx index 6a189b30..4c24268c 100644 --- a/admin-ui/src/features/book/match-dialog.tsx +++ b/admin-ui/src/features/book/match-dialog.tsx @@ -40,7 +40,7 @@ import { formatDuration } from '@/lib/format'; import { toast } from '@/lib/toast'; import { cn } from '@/lib/utils'; import { regionTag } from '@/lib/regions'; -import { displayValue } from './book-model'; +import { shownValue } from './book-model'; import { acceptRequest, candidateAuthors, @@ -659,7 +659,7 @@ function CompareTable({ )} > {r.mine ? ( - clip(displayValue(r.field, r.mine), 120) + clip(shownValue(t, r.field, r.mine), 120) ) : ( {t('book.match.empty')} )} @@ -671,7 +671,7 @@ function CompareTable({ r.same ? ( {t('book.match.same')} ) : r.theirs ? ( - clip(displayValue(r.field, r.theirs), 150) + clip(shownValue(t, r.field, r.theirs), 150) ) : ( {t('book.match.notProvided')} ) diff --git a/admin-ui/src/features/book/match-model.test.ts b/admin-ui/src/features/book/match-model.test.ts index fdbb60e3..bcf91bdd 100644 --- a/admin-ui/src/features/book/match-model.test.ts +++ b/admin-ui/src/features/book/match-model.test.ts @@ -72,18 +72,35 @@ describe('compare', () => { it('maps a candidate onto the fields', () => { expect(communityValues(candidate(), recording())).toEqual({ title: 'The Way of Kings', + subtitle: '', author: 'Brandon Sanderson', narrator: 'Michael Kramer, Kate Reading', series: 'The Stormlight Archive', series_index: '1', more_series: '', published: '2010', + publisher: 'Macmillan Audio', + language: '', + abridged: '', + genres: '', + tags: '', description: 'Roshar is a world of stone and storms.', asin: 'B003P2WO5E', isbn: '', }); }); + it('offers the subtitle, the language and abridged only when the recording says so', () => { + const c = candidate({ subtitle: 'Book One', language: 'en' }); + const values = communityValues(c, recording({ abridged: true })); + expect(values).toMatchObject({ subtitle: 'Book One', language: 'en', abridged: 'true' }); + // The community's false is also "not recorded". + expect(communityValues(c, recording({ abridged: false })).abridged).toBe(''); + // The label fields have no row: the community offers neither. + const rows = compareRows(bookDetail().fields, c, recording()); + expect(rows.some((r) => r.field === 'genres' || r.field === 'tags')).toBe(false); + }); + it('offers a work in several series as one main series and the rest', () => { const watch = 'Discworld: Ankh-Morpork City Watch'; const c = candidate({ @@ -194,13 +211,17 @@ describe('compare', () => { detail.fields.narrator = { ...detail.fields.narrator, source: 'edited', locked: true }; const rows = compareRows(detail.fields, candidate(), recording()); const offered = rows.filter((r) => r.offered).map((r) => r.field); - expect(offered).toEqual(['narrator', 'published', 'description']); + expect(offered).toEqual(['narrator', 'published', 'publisher', 'description']); expect(rows.find((r) => r.field === 'title')?.same).toBe(true); expect(rows.find((r) => r.field === 'isbn')).toMatchObject({ same: false, offered: false }); const ticks = defaultTicks(rows); - expect([...ticks]).toEqual(['published', 'description']); + expect([...ticks]).toEqual(['published', 'publisher', 'description']); expect(acceptRequest(rows, ticks)).toEqual({ - set: { published: '2010', description: 'Roshar is a world of stone and storms.' }, + set: { + published: '2010', + publisher: 'Macmillan Audio', + description: 'Roshar is a world of stone and storms.', + }, source: 'community', }); }); diff --git a/admin-ui/src/features/book/match-model.ts b/admin-ui/src/features/book/match-model.ts index b407c1ac..dce91a8a 100644 --- a/admin-ui/src/features/book/match-model.ts +++ b/admin-ui/src/features/book/match-model.ts @@ -1,4 +1,5 @@ import { + LABEL_FIELDS, OVERRIDE_FIELDS, type BookEditRequest, type FieldSource, @@ -51,7 +52,9 @@ export function candidateAuthors(c: MatchCandidate): string { } /** - * What the community says for each field, from the work and the chosen recording. + * What the community says for each field, from the work and the chosen recording + * (as matchrun.Propose: the work's subtitle and language, the recording's + * publisher, abridged only when it is marked so). * A work in several series offers one as the main series - the one the book is * filed under (`mainSeries`) when the work is in it, else the recording's first * (its own edition, when the community lists one) - and the rest as the book's other series (as matchrun.planSeries @@ -70,12 +73,20 @@ export function communityValues( const year = (c.first_published || rec?.release_date || '').match(/^\d{4}/)?.[0]; return { title: clean('title', c.title), + subtitle: clean('subtitle', c.subtitle), author: clean('author', candidateAuthors(c)), narrator: clean('narrator', (rec?.narrators ?? []).map((n) => n.name).join(', ')), series: clean('series', series?.name), series_index: clean('series_index', series?.position), more_series: clean('more_series', otherSeries(others)), published: clean('published', year), + publisher: clean('publisher', rec?.publisher), + language: clean('language', c.language), + // The community's false is also "not recorded": only true says anything. + abridged: rec?.abridged ? 'true' : '', + // A match candidate carries no genres, and the community has no tags. + genres: '', + tags: '', description: clean('description', c.description), asin: clean('asin', rec?.asins?.[0]), isbn: clean('isbn', rec?.isbns?.[0]), @@ -102,13 +113,18 @@ export interface CompareRow { offered: boolean; } +/** + * The compare table's rows, in field order. The label fields are left out: the + * community offers neither, so their rows would only ever say so. + */ export function compareRows( fields: Record, c: MatchCandidate, rec: MatchRecording | undefined, ): CompareRow[] { const theirs = communityValues(c, rec, fields.series.value); - return OVERRIDE_FIELDS.map((f) => { + const offerable = OVERRIDE_FIELDS.filter((f) => !(LABEL_FIELDS as readonly string[]).includes(f)); + return offerable.map((f) => { const mine = fields[f].value; const same = !!theirs[f] && clean(f, mine) === theirs[f]; return { diff --git a/admin-ui/src/features/book/save-changes.tsx b/admin-ui/src/features/book/save-changes.tsx index 1315029a..d365c8f9 100644 --- a/admin-ui/src/features/book/save-changes.tsx +++ b/admin-ui/src/features/book/save-changes.tsx @@ -18,7 +18,7 @@ import { import { toastError } from '@/lib/errors'; import { toast } from '@/lib/toast'; import { cn } from '@/lib/utils'; -import { checkField, diffRows, displayValue, saveRequest, type Drafts } from './book-model'; +import { checkField, diffRows, saveRequest, shownValue, type Drafts } from './book-model'; /** * The floating save bar (STYLEGUIDE.md "Save bar"): how many edits are waiting, @@ -164,8 +164,8 @@ function DiffBody({ - - + + ))} diff --git a/admin-ui/src/features/libraries/libraries.test.tsx b/admin-ui/src/features/libraries/libraries.test.tsx index 822de8c6..84716049 100644 --- a/admin-ui/src/features/libraries/libraries.test.tsx +++ b/admin-ui/src/features/libraries/libraries.test.tsx @@ -280,9 +280,43 @@ describe('libraries', () => { expect(await screen.findByText('audiosilo-fiction-2026-10-03.json')).toBeInTheDocument(); const exp = calls.find((c) => c.path === '/admin/libraries/1/export'); expect(exp?.headers.Authorization).toBe('Bearer stored'); + // The plain export names no paths. + expect(exp?.query.get('include')).toBeNull(); expect(createObjectURL).toHaveBeenCalled(); }); + it('exports with file paths only after a warning', async () => { + vi.stubGlobal( + 'URL', + Object.assign(URL, { createObjectURL: vi.fn(() => 'blob:export'), revokeObjectURL: vi.fn() }), + ); + const calls = mockFetch( + routes({ + 'GET /admin/libraries/1/export': { + raw: '{"format":"audiosilo-books"}', + headers: { + 'Content-Type': 'application/json', + 'Content-Disposition': 'attachment; filename="audiosilo-fiction-2026-10-03.json"', + }, + }, + }), + ); + renderApp('/library/libraries'); + const user = userEvent.setup(); + const fiction = await screen.findByRole('listitem', { name: 'Fiction' }); + await user.click(within(fiction).getByRole('button', { name: 'Actions for Fiction' })); + await user.click(await screen.findByRole('menuitem', { name: 'Export with file paths...' })); + const dialog = await screen.findByRole('dialog', { name: 'Export Fiction with file paths' }); + expect( + within(dialog).getByText(/Don't attach this file to a public issue/), + ).toBeInTheDocument(); + expect(calls.some((c) => c.path === '/admin/libraries/1/export')).toBe(false); + await user.click(within(dialog).getByRole('button', { name: 'Export with paths' })); + expect(await screen.findByText('audiosilo-fiction-2026-10-03.json')).toBeInTheDocument(); + const exp = calls.find((c) => c.path === '/admin/libraries/1/export'); + expect(exp?.query.get('include')).toBe('paths'); + }); + it('shows the empty state with no libraries', async () => { mockFetch(routes({ 'GET /admin/libraries': { body: { libraries: [] } } })); renderApp('/library/libraries'); diff --git a/admin-ui/src/features/libraries/library-card.tsx b/admin-ui/src/features/libraries/library-card.tsx index a4c1cdc9..20332bd6 100644 --- a/admin-ui/src/features/libraries/library-card.tsx +++ b/admin-ui/src/features/libraries/library-card.tsx @@ -10,18 +10,22 @@ import { Clock, Download, Ellipsis, + FileWarning, FolderTree, GripVertical, Pencil, RefreshCw, Split, Trash2, + TriangleAlert, Unplug, } from 'lucide-react'; import { downloadLibraryExport } from '@/api/client'; import { useRecentBooks } from '@/api/hooks'; import type { AdminLibrary, ScanProgress } from '@/api/types'; import { BookCover } from '@/components/book-cover'; +import { ConfirmDialog } from '@/components/confirm-dialog'; +import { Notice } from '@/components/notice'; import { ProgressBar } from '@/components/progress-bar'; import { Badge } from '@/components/ui/badge'; import { Button, buttonVariants } from '@/components/ui/button'; @@ -43,7 +47,7 @@ import { OfflineNotice } from './offline-notice'; import { rescanLibrary } from './rescan'; import { scheduleLabel } from './scan-settings'; -type OpenDialog = 'edit' | 'detect' | 'delete' | null; +type OpenDialog = 'edit' | 'detect' | 'delete' | 'exportPaths' | null; /** One library: covers, name, status, folder, and everything an admin does to it. */ export function LibraryCard({ @@ -66,10 +70,12 @@ export function LibraryCard({ // Waiting in the job queue behind another library's scan (a running one wins). const queued = !running && !!l.scan.queued; - const exportBooks = async () => { + // With paths, each row names its library and its books' paths: a file for the + // admin's own scripts (the menu warns before it is made), never the default. + const exportBooks = async (paths = false) => { setExporting(true); try { - const file = await downloadLibraryExport(l.id); + const file = await downloadLibraryExport(l.id, { paths }); toast.add({ title: t('libraries.toast.exported', { name: l.name }), description: file, @@ -212,6 +218,10 @@ export function LibraryCard({