diff --git a/internal/datasource/wikidata/gowikidata.go b/internal/datasource/wikidata/gowikidata.go index 4ce3a752..6c73eda8 100644 --- a/internal/datasource/wikidata/gowikidata.go +++ b/internal/datasource/wikidata/gowikidata.go @@ -6,25 +6,47 @@ type Gowikidata struct { } func (w Gowikidata) NewSearch(search string, language string) (SearchEntitiesRequest, error) { - return gowikidata.NewSearch(search, language) + req, err := gowikidata.NewSearch(search, language) + if err != nil { + return nil, err + } + return searchRequest{url: req.URL}, nil } func (w Gowikidata) NewGetEntities(ids []string) (GetEntitiesRequest, error) { request, err := gowikidata.NewGetEntities(ids) + if err != nil { + return nil, err + } + return entitiesRequest{req: request}, nil +} + +type searchRequest struct { + url string +} - return EntitiesRequest{request}, err +func (s searchRequest) Get() (*gowikidata.SearchEntitiesResponse, error) { + var response gowikidata.SearchEntitiesResponse + if err := apiHTTPClient.getJSON(s.url, &response); err != nil { + return nil, err + } + return &response, nil } -type EntitiesRequest struct { +type entitiesRequest struct { req *gowikidata.WikiDataGetEntitiesRequest } -func (e EntitiesRequest) SetProps(props []string) { +func (e entitiesRequest) SetProps(props []string) { e.req.SetProps(props) } -func (e EntitiesRequest) SetLanguages(languages []string) { +func (e entitiesRequest) SetLanguages(languages []string) { e.req.SetLanguages(languages) } -func (e EntitiesRequest) Get() (*map[string]gowikidata.Entity, error) { - return e.req.Get() +func (e entitiesRequest) Get() (*map[string]gowikidata.Entity, error) { + var response gowikidata.GetEntitiesResponse + if err := apiHTTPClient.getJSON(e.req.URL, &response); err != nil { + return nil, err + } + return &response.Entities, nil } diff --git a/internal/datasource/wikidata/httpclient.go b/internal/datasource/wikidata/httpclient.go new file mode 100644 index 00000000..18d90a0f --- /dev/null +++ b/internal/datasource/wikidata/httpclient.go @@ -0,0 +1,114 @@ +package wikidata + +import ( + "encoding/json" + "fmt" + "io" + "log" + "net/http" + "strconv" + "sync" + "time" +) + +const defaultRetryAfter = 60 * time.Second + +var apiHTTPClient = newRateLimitedHTTPClient() + +type rateLimitedHTTPClient struct { + mu sync.Mutex + resumeAt time.Time + client *http.Client +} + +func newRateLimitedHTTPClient() *rateLimitedHTTPClient { + return &rateLimitedHTTPClient{ + client: &http.Client{}, + } +} + +func (c *rateLimitedHTTPClient) getJSON(url string, dest any) error { + for { + c.waitUntil("") + + req, err := http.NewRequest(http.MethodGet, url, nil) + if err != nil { + return err + } + for key, value := range wikidataAPIHeaders() { + req.Header.Set(key, value) + } + + resp, err := c.client.Do(req) + if err != nil { + return err + } + + if resp.StatusCode == http.StatusTooManyRequests { + _ = resp.Body.Close() + c.waitUntil(resp.Header.Get("Retry-After")) + continue + } + + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + _ = resp.Body.Close() + return fmt.Errorf("request failed with status code %d: %s", resp.StatusCode, body) + } + + body, err := io.ReadAll(resp.Body) + _ = resp.Body.Close() + if err != nil { + return err + } + return json.Unmarshal(body, dest) + } +} + +// waitUntil blocks until the shared pause deadline has passed. When retryAfterHeader +// is non-empty, the deadline is extended using the Retry-After header value. +func (c *rateLimitedHTTPClient) waitUntil(retryAfterHeader string) { + c.mu.Lock() + var rateLimitWait time.Duration + if retryAfterHeader != "" { + rateLimitWait = parseRetryAfter(retryAfterHeader) + resumeAt := time.Now().Add(rateLimitWait) + if resumeAt.After(c.resumeAt) { + c.resumeAt = resumeAt + } + } + wait := time.Until(c.resumeAt) + c.mu.Unlock() + if retryAfterHeader != "" { + log.Printf("Wikidata rate limit reached, waiting %s before retrying", rateLimitWait) + } + if wait > 0 { + time.Sleep(wait) + } +} + +func parseRetryAfter(header string) time.Duration { + if header == "" { + return defaultRetryAfter + } + if seconds, err := strconv.Atoi(header); err == nil { + if seconds <= 0 { + return defaultRetryAfter + } + return time.Duration(seconds) * time.Second + } + if retryTime, err := http.ParseTime(header); err == nil { + wait := time.Until(retryTime) + if wait <= 0 { + return defaultRetryAfter + } + return wait + } + return defaultRetryAfter +} + +func wikidataAPIHeaders() map[string]string { + return map[string]string{ + "User-Agent": "Mozilla/5.0 (compatible; coreander/1.0; +https://github.com/svera/coreander)", + } +} diff --git a/internal/datasource/wikidata/httpclient_test.go b/internal/datasource/wikidata/httpclient_test.go new file mode 100644 index 00000000..61fcfa30 --- /dev/null +++ b/internal/datasource/wikidata/httpclient_test.go @@ -0,0 +1,93 @@ +package wikidata + +import ( + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + "time" +) + +func TestParseRetryAfter(t *testing.T) { + t.Run("seconds", func(t *testing.T) { + if got := parseRetryAfter("30"); got != 30*time.Second { + t.Fatalf("expected 30s, got %s", got) + } + }) + + t.Run("http date", func(t *testing.T) { + retryTime := time.Now().Add(45 * time.Second).UTC() + got := parseRetryAfter(retryTime.Format(http.TimeFormat)) + if got < 44*time.Second || got > 46*time.Second { + t.Fatalf("expected about 45s, got %s", got) + } + }) + + t.Run("missing header uses default", func(t *testing.T) { + if got := parseRetryAfter(""); got != defaultRetryAfter { + t.Fatalf("expected default %s, got %s", defaultRetryAfter, got) + } + }) +} + +func TestRateLimitedHTTPClientWaitsForRetryAfter(t *testing.T) { + var requests atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if requests.Add(1) == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(`{"searchinfo":[],"search":[]}`)) + })) + defer server.Close() + + client := newRateLimitedHTTPClient() + client.client = server.Client() + + start := time.Now() + var payload map[string]any + if err := client.getJSON(server.URL, &payload); err != nil { + t.Fatal(err) + } + if requests.Load() != 2 { + t.Fatalf("expected 2 requests, got %d", requests.Load()) + } + if elapsed := time.Since(start); elapsed < time.Second { + t.Fatalf("expected to wait at least 1s after 429, took %s", elapsed) + } +} + +func TestRateLimitedHTTPClientBlocksConcurrentRequests(t *testing.T) { + var requests atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + count := requests.Add(1) + if count == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(`{"entities":{}}`)) + })) + defer server.Close() + + client := newRateLimitedHTTPClient() + client.client = server.Client() + + start := time.Now() + done := make(chan struct{}, 2) + for range 2 { + go func() { + var payload map[string]any + _ = client.getJSON(server.URL, &payload) + done <- struct{}{} + }() + } + <-done + <-done + if elapsed := time.Since(start); elapsed < time.Second { + t.Fatalf("expected concurrent requests to honor shared wait, took %s", elapsed) + } +} diff --git a/internal/datasource/wikidata/mockserver.go b/internal/datasource/wikidata/mockserver.go index 43895b87..3b3137a0 100644 --- a/internal/datasource/wikidata/mockserver.go +++ b/internal/datasource/wikidata/mockserver.go @@ -23,6 +23,10 @@ func NewMockServer(t *testing.T, fixturePath string) *httptest.Server { } if queryValues.Get("action") == "wbgetentities" { id := queryValues.Get("ids") + if strings.Contains(id, "|") { + parts := strings.Split(id, "|") + id = parts[0] + } returnResponse(fmt.Sprintf("wbgetentities-%s", id), w, fixturePath) return } diff --git a/internal/datasource/wikidata/service.go b/internal/datasource/wikidata/service.go index 9b1e786a..de8454d6 100644 --- a/internal/datasource/wikidata/service.go +++ b/internal/datasource/wikidata/service.go @@ -19,6 +19,10 @@ import ( const imgUrl = "https://upload.wikimedia.org/wikipedia/commons/%s/%s/%s" +const maxEntitiesPerRequest = 50 + +var entityFetchProps = []string{"descriptions", "claims", "sitelinks/urls", "labels"} + type wikidata interface { NewSearch(string, string) (SearchEntitiesRequest, error) NewGetEntities([]string) (GetEntitiesRequest, error) @@ -43,7 +47,7 @@ func NewWikidataSource(w wikidata) WikidataSource { } func (a WikidataSource) SearchAuthor(name string, languages []string) (model.Author, error) { - ids, err := a.getEntityIds(name) + ids, err := a.SearchEntityIDs(name) if err != nil { return nil, err } @@ -55,67 +59,183 @@ func (a WikidataSource) SearchAuthor(name string, languages []string) (model.Aut return a.RetrieveAuthor(ids, languages) } -// RetrieveAuthor returns the first match from the list of passed Wikidata entity IDs that represents a human -func (a WikidataSource) RetrieveAuthor(ids []string, languages []string) (model.Author, error) { - author := Author{ - wikipediaLink: make(map[string]string), - description: make(map[string]string), +// SearchEntityIDs returns Wikidata entity IDs matching the given author name. +func (a WikidataSource) SearchEntityIDs(name string) ([]string, error) { + query, err := a.wikidata.NewSearch(url.QueryEscape(name), "en") + if err != nil { + return nil, err + } + result, err := query.Get() + if err != nil { + return nil, err + } + + if len(result.SearchResult) == 0 { + return nil, nil } + ids := make([]string, 0, len(result.SearchResult)) + for _, entity := range result.SearchResult { + ids = append(ids, entity.ID) + } + return ids, nil +} + +// RetrieveAuthor returns the first match from the list of passed Wikidata entity IDs that represents a human +func (a WikidataSource) RetrieveAuthor(ids []string, languages []string) (model.Author, error) { for _, id := range ids { if !validateID(id) { return Author{}, fmt.Errorf("invalid author ID %s", id) } } - entitiesReq, err := a.wikidata.NewGetEntities(ids) + entities, err := a.fetchEntitiesBatched(ids, languages, 0) if err != nil { return nil, err } - entitiesReq.SetProps([]string{"descriptions", "claims", "sitelinks/urls", "labels"}) - entitiesReq.SetLanguages(languages) - // Call get to make the request based on the configurations - entities, err := entitiesReq.Get() + + author, err := a.authorFromEntityIDs(ids, entities, languages) if err != nil { return nil, err } - - author.wikidataEntityId, author.instanceOf = getMostAccurateID(ids, entities) if author.instanceOf == InstanceUnknown { return nil, nil } + return author, nil +} + +// RetrieveAuthors fetches metadata for multiple authors in as few Wikidata requests as possible. +// Keys are caller-defined identifiers (for example author slugs). Values are Wikidata entity ID +// candidates for each author, in search-result order. +func (a WikidataSource) RetrieveAuthors(candidates map[string][]string, languages []string, batchInterval time.Duration) (map[string]model.Author, error) { + if len(candidates) == 0 { + return map[string]model.Author{}, nil + } + + uniqueIDs := make([]string, 0) + seen := make(map[string]struct{}) + for _, ids := range candidates { + for _, id := range ids { + if !validateID(id) { + return nil, fmt.Errorf("invalid author ID %s", id) + } + if id == "" { + continue + } + if _, ok := seen[id]; ok { + continue + } + seen[id] = struct{}{} + uniqueIDs = append(uniqueIDs, id) + } + } + + entities, err := a.fetchEntitiesBatched(uniqueIDs, languages, batchInterval) + if err != nil { + return nil, err + } - if value, exists := (*entities)[author.wikidataEntityId].Claims[propertyBirthName]; exists { + results := make(map[string]model.Author, len(candidates)) + for key, ids := range candidates { + if len(ids) == 0 { + continue + } + author, err := a.authorFromEntityIDs(ids, entities, languages) + if err != nil { + return nil, err + } + if author.instanceOf == InstanceUnknown { + continue + } + results[key] = author + } + return results, nil +} + +func (a WikidataSource) fetchEntitiesBatched(ids []string, languages []string, batchInterval time.Duration) (map[string]gowikidata.Entity, error) { + entities := make(map[string]gowikidata.Entity, len(ids)) + if len(ids) == 0 { + return entities, nil + } + + for start := 0; start < len(ids); start += maxEntitiesPerRequest { + if start > 0 && batchInterval > 0 { + time.Sleep(batchInterval) + } + end := start + maxEntitiesPerRequest + if end > len(ids) { + end = len(ids) + } + chunk := ids[start:end] + + entitiesReq, err := a.wikidata.NewGetEntities(chunk) + if err != nil { + return nil, err + } + entitiesReq.SetProps(entityFetchProps) + entitiesReq.SetLanguages(languages) + batch, err := entitiesReq.Get() + if err != nil { + return nil, err + } + for id, entity := range *batch { + entities[id] = entity + } + } + return entities, nil +} + +func (a WikidataSource) authorFromEntityIDs(ids []string, entities map[string]gowikidata.Entity, languages []string) (Author, error) { + author := Author{ + wikipediaLink: make(map[string]string), + description: make(map[string]string), + } + if len(ids) == 0 { + return author, nil + } + + entityPtr := &entities + author.wikidataEntityId, author.instanceOf = getMostAccurateID(ids, entityPtr) + if author.instanceOf == InstanceUnknown { + return author, nil + } + + entity, ok := entities[author.wikidataEntityId] + if !ok { + return author, nil + } + + if value, exists := entity.Claims[propertyBirthName]; exists { author.birthName = value[0].MainSnak.DataValue.Value.ValueFields.Text - } else if value, exists := (*entities)[author.wikidataEntityId].Claims[propertyNameInNativeLanguage]; exists { + } else if value, exists := entity.Claims[propertyNameInNativeLanguage]; exists { author.birthName = value[0].MainSnak.DataValue.Value.ValueFields.Text - } else if value, exists := (*entities)[author.wikidataEntityId].Claims[propertyOfficialName]; exists { + } else if value, exists := entity.Claims[propertyOfficialName]; exists { author.birthName = value[0].MainSnak.DataValue.Value.ValueFields.Text } - if value, exists := (*entities)[author.wikidataEntityId].Claims[propertySexOrGender]; exists { + if value, exists := entity.Claims[propertySexOrGender]; exists { author.gender = parseGender(value[0]) } author.retrievedOn = time.Now().UTC() for _, lang := range languages { - if url := (*entities)[author.wikidataEntityId].SiteLinks[fmt.Sprintf("%swiki", lang)].URL; url != "" { + if url := entity.SiteLinks[fmt.Sprintf("%swiki", lang)].URL; url != "" { author.wikipediaLink[lang] = url } - if description := (*entities)[author.wikidataEntityId].Descriptions[lang].Value; description != "" { + if description := entity.Descriptions[lang].Value; description != "" { author.description[lang] = description } } - if claim, exists := (*entities)[author.wikidataEntityId].Claims[propertyDateOfBirth]; exists { + if claim, exists := entity.Claims[propertyDateOfBirth]; exists { author.dateOfBirth = parseDate(claim) } - if claim, exists := (*entities)[author.wikidataEntityId].Claims[propertyDateOfDeath]; exists { + if claim, exists := entity.Claims[propertyDateOfDeath]; exists { author.dateOfDeath = parseDate(claim) } - if value, exists := (*entities)[author.wikidataEntityId].Claims[propertyWebsite]; exists { + if value, exists := entity.Claims[propertyWebsite]; exists { author.website = value[0].MainSnak.DataValue.Value.S } - if value, exists := (*entities)[author.wikidataEntityId].Claims[propertyPseudonym]; exists { + if value, exists := entity.Claims[propertyPseudonym]; exists { author.pseudonyms = make([]string, 0, len(value)) for _, claim := range value { pseudonym, err := strconv.Unquote("\"" + claim.MainSnak.DataValue.Value.S + "\"") @@ -126,10 +246,10 @@ func (a WikidataSource) RetrieveAuthor(ids []string, languages []string) (model. } } - if value, exists := (*entities)[author.wikidataEntityId].Claims[propertyImage]; exists { + if value, exists := entity.Claims[propertyImage]; exists { img, err := strconv.Unquote("\"" + value[0].MainSnak.DataValue.Value.S + "\"") if err != nil { - return nil, err + return Author{}, err } if slices.Contains([]string{".png", ".jpg", ".jpeg", ".tif", ".tiff"}, strings.ToLower(filepath.Ext(img))) { @@ -158,29 +278,6 @@ func getMostAccurateID(ids []string, entities *map[string]gowikidata.Entity) (st return "", InstanceUnknown } -// getEntityIds return all entity IDs from Wikidata which matches the passed name -func (a WikidataSource) getEntityIds(name string) ([]string, error) { - query, err := a.wikidata.NewSearch(url.QueryEscape(name), "en") - if err != nil { - return []string{}, err - } - result, err := query.Get() - if err != nil { - return []string{}, err - } - - if len(result.SearchResult) == 0 { - return []string{}, nil - } - - res := make([]string, 0, len(result.SearchResult)) - for _, entity := range result.SearchResult { - res = append(res, entity.ID) - } - - return res, nil -} - func parseGender(claim gowikidata.Claim) float64 { switch claim.MainSnak.DataValue.Value.ValueFields.ID { case qidGenderMale: diff --git a/internal/datasource/wikidata/service_test.go b/internal/datasource/wikidata/service_test.go index bfb63f5c..64f25be6 100644 --- a/internal/datasource/wikidata/service_test.go +++ b/internal/datasource/wikidata/service_test.go @@ -71,3 +71,23 @@ func testCases(t *testing.T) []testCase { }, } } + +func TestRetrieveAuthorsBatch(t *testing.T) { + mockServer := NewMockServer(t, "fixtures") + defer mockServer.Close() + gowikidata.WikidataDomain = mockServer.URL + + source := NewWikidataSource(Gowikidata{}) + authors, err := source.RetrieveAuthors(map[string][]string{ + "miguel": {"Q1234"}, + }, []string{"en"}, 0) + if err != nil { + t.Fatal(err) + } + if len(authors) != 1 { + t.Fatalf("expected 1 author from batch retrieve, got %d", len(authors)) + } + if authors["miguel"].SourceID() != "Q1234" { + t.Fatalf("expected Q1234, got %q", authors["miguel"].SourceID()) + } +} diff --git a/internal/index/author_search.go b/internal/index/author_search.go new file mode 100644 index 00000000..1cdd492f --- /dev/null +++ b/internal/index/author_search.go @@ -0,0 +1,215 @@ +package index + +import ( + "cmp" + "slices" + "strings" + + "github.com/blevesearch/bleve/v2" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/search/query" + "github.com/rickb777/date/v2" + "github.com/svera/coreander/v5/internal/result" +) + +type AuthorSearchFields struct { + Name string + Gender *float64 + BirthDateFrom date.Date + BirthDateTo date.Date + DeathDateFrom date.Date + DeathDateTo date.Date + SortBy []string +} + +func (b *BleveIndexer) SearchAuthors(searchFields AuthorSearchFields, page, resultsPerPage int) (result.Paginated[[]Author], error) { + filtersQuery := bleve.NewConjunctionQuery() + + if q := b.authorNameQuery(searchFields.Name); q != nil { + filtersQuery.AddQuery(q) + } else { + filtersQuery.AddQuery(bleve.NewMatchAllQuery()) + } + + addAuthorFilters(searchFields, filtersQuery) + + return b.runAuthorsPaginatedQuery(filtersQuery, page, resultsPerPage, searchFields.SortBy) +} + +func (b *BleveIndexer) authorNameQuery(name string) query.Query { + name = strings.TrimSpace(name) + if name == "" { + return nil + } + + disj := bleve.NewDisjunctionQuery() + terms := b.analyzedAuthorNameTerms(name) + + for _, field := range []string{"Name", "BirthName"} { + match := bleve.NewMatchQuery(name) + match.SetField(field) + match.Analyzer = defaultAnalyzer + match.Operator = query.MatchQueryOperatorAnd + disj.AddQuery(match) + + if len(terms) != 1 { + continue + } + + term := terms[0] + prefix := bleve.NewPrefixQuery(term) + prefix.SetField(field) + disj.AddQuery(prefix) + + wildcard := bleve.NewWildcardQuery("*" + escapeWildcard(term) + "*") + wildcard.SetField(field) + disj.AddQuery(wildcard) + } + return disj +} + +func (b *BleveIndexer) analyzedAuthorNameTerms(name string) []string { + analyzer := b.authorsIdx.Mapping().AnalyzerNamed(defaultAnalyzer) + if analyzer == nil { + return nil + } + + tokens := analyzer.Analyze([]byte(name)) + terms := make([]string, 0, len(tokens)) + for _, token := range tokens { + if token == nil || len(token.Term) == 0 { + continue + } + terms = append(terms, string(token.Term)) + } + return terms +} + +func escapeWildcard(value string) string { + replacer := strings.NewReplacer(`\`, `\\`, `*`, `\*`, `?`, `\?`) + return replacer.Replace(value) +} + +func addAuthorFilters(searchFields AuthorSearchFields, filtersQuery *query.ConjunctionQuery) { + if searchFields.Gender != nil { + min := *searchFields.Gender + max := min + 1 + q := bleve.NewNumericRangeQuery(&min, &max) + q.SetField("Gender") + filtersQuery.AddQuery(q) + } + addDateRangeFilter(filtersQuery, "DateOfBirth.Date", searchFields.BirthDateFrom, searchFields.BirthDateTo) + addDeathDateRangeFilter(filtersQuery, searchFields.DeathDateFrom, searchFields.DeathDateTo) +} + +func addDeathDateRangeFilter(filtersQuery *query.ConjunctionQuery, from, to date.Date) { + if from == 0 && to == 0 { + return + } + addDateRangeFilter(filtersQuery, "DateOfDeath.Date", from, to) + + // Living authors are indexed with DateOfDeath.Date == 0, which otherwise matches + // an open-ended "died before" filter because 0 <= max. + zero := float64(0) + one := float64(1) + maxExclusive := false + zeroDeathDate := bleve.NewNumericRangeInclusiveQuery(&zero, &one, nil, &maxExclusive) + zeroDeathDate.SetField("DateOfDeath.Date") + excludeLiving := bleve.NewBooleanQuery() + excludeLiving.AddMustNot(zeroDeathDate) + filtersQuery.AddQuery(excludeLiving) +} + +func (b *BleveIndexer) runAuthorsPaginatedQuery(q query.Query, page, resultsPerPage int, sortBy []string) (result.Paginated[[]Author], error) { + if page < 1 { + page = 1 + } + + if desc, ok := authorDocumentCountSortDesc(sortBy); ok { + countResult, err := b.authorsIdx.Search(bleve.NewSearchRequestOptions(q, 0, 0, false)) + if err != nil { + return result.Paginated[[]Author]{}, err + } + total := int(countResult.Total) + if total == 0 { + return result.Paginated[[]Author]{}, nil + } + + searchOptions := bleve.NewSearchRequestOptions(q, total, 0, false) + searchOptions.Fields = []string{"*"} + searchResult, err := b.authorsIdx.Search(searchOptions) + if err != nil { + return result.Paginated[[]Author]{}, err + } + + authors := hydrateAuthors(searchResult.Hits) + counts, err := b.DocumentCountsByAuthorSlugs(authorSlugsFromAuthors(authors)) + if err != nil { + return result.Paginated[[]Author]{}, err + } + sortAuthorsByDocumentCount(authors, counts, desc) + return result.Paginate(resultsPerPage, page, total, authors), nil + } + + searchOptions := bleve.NewSearchRequestOptions(q, resultsPerPage, (page-1)*resultsPerPage, false) + searchOptions.SortBy(sortBy) + searchOptions.Fields = []string{"*"} + searchResult, err := b.authorsIdx.Search(searchOptions) + if err != nil { + return result.Paginated[[]Author]{}, err + } + if searchResult.Total == 0 { + return result.Paginated[[]Author]{}, nil + } + + return result.Paginate( + resultsPerPage, + page, + int(searchResult.Total), + hydrateAuthors(searchResult.Hits), + ), nil +} + +func authorDocumentCountSortDesc(sortBy []string) (desc bool, ok bool) { + if len(sortBy) != 1 { + return false, false + } + switch sortBy[0] { + case "DocumentCount": + return false, true + case "-DocumentCount": + return true, true + default: + return false, false + } +} + +func hydrateAuthors(hits search.DocumentMatchCollection) []Author { + authors := make([]Author, len(hits)) + for i, hit := range hits { + authors[i] = hydrateAuthor(hit) + } + return authors +} + +func authorSlugsFromAuthors(authors []Author) []string { + slugs := make([]string, len(authors)) + for i, author := range authors { + slugs[i] = author.Slug + } + return slugs +} + +func sortAuthorsByDocumentCount(authors []Author, counts map[string]uint64, desc bool) { + slices.SortFunc(authors, func(a, b Author) int { + countA := counts[a.Slug] + countB := counts[b.Slug] + if countA != countB { + if desc { + return cmp.Compare(countB, countA) + } + return cmp.Compare(countA, countB) + } + return strings.Compare(a.Name, b.Name) + }) +} diff --git a/internal/index/author_search_test.go b/internal/index/author_search_test.go new file mode 100644 index 00000000..6fdd6bfa --- /dev/null +++ b/internal/index/author_search_test.go @@ -0,0 +1,212 @@ +package index_test + +import ( + "testing" + "time" + + "github.com/blevesearch/bleve/v2" + "github.com/rickb777/date/v2" + "github.com/svera/coreander/v5/internal/datasource/wikidata" + "github.com/svera/coreander/v5/internal/index" + "github.com/svera/coreander/v5/internal/metadata" + "github.com/svera/coreander/v5/internal/precisiondate" +) + +func TestSearchAuthors(t *testing.T) { + authorsIndexMem, err := bleve.NewMemOnly(index.CreateAuthorsMapping()) + if err != nil { + t.Fatal(err) + } + documentsIndexMem, err := bleve.NewMemOnly(index.CreateDocumentsMapping()) + if err != nil { + t.Fatal(err) + } + + idx := index.NewBleve(documentsIndexMem, authorsIndexMem, nil, "", nil, index.Config{}) + + authors := []index.Author{ + { + Slug: "george-orwell", + Name: "George Orwell", + BirthName: "Eric Arthur Blair", + Gender: float64(wikidata.GenderMale), + DateOfBirth: precisiondate.NewPrecisionDate("+1903-06-25T00:00:00Z", precisiondate.PrecisionDay), + DateOfDeath: precisiondate.NewPrecisionDate("+1950-01-21T00:00:00Z", precisiondate.PrecisionDay), + RetrievedOn: mustParseTime("2020-01-01T00:00:00Z"), + }, + { + Slug: "jane-austen", + Name: "Jane Austen", + Gender: float64(wikidata.GenderFemale), + DateOfBirth: precisiondate.NewPrecisionDate("+1775-12-16T00:00:00Z", precisiondate.PrecisionDay), + DateOfDeath: precisiondate.NewPrecisionDate("+1817-07-18T00:00:00Z", precisiondate.PrecisionDay), + RetrievedOn: mustParseTime("2020-01-01T00:00:00Z"), + }, + { + Slug: "arturo-perez-reverte", + Name: "Arturo Pérez-Reverte", + Gender: float64(wikidata.GenderMale), + DateOfBirth: precisiondate.NewPrecisionDate("+1951-11-24T00:00:00Z", precisiondate.PrecisionDay), + RetrievedOn: mustParseTime("2020-01-01T00:00:00Z"), + }, + { + Slug: "living-author", + Name: "Living Author", + Gender: float64(wikidata.GenderMale), + DateOfBirth: precisiondate.NewPrecisionDate("+1990-01-01T00:00:00Z", precisiondate.PrecisionDay), + RetrievedOn: mustParseTime("2020-01-01T00:00:00Z"), + }, + } + for _, author := range authors { + if err := idx.IndexAuthor(author); err != nil { + t.Fatal(err) + } + } + + t.Run("by name", func(t *testing.T) { + res, err := idx.SearchAuthors(index.AuthorSearchFields{Name: "Orwell"}, 1, 10) + if err != nil { + t.Fatal(err) + } + hits := res.Hits() + if res.TotalHits() != 1 || hits[0].Slug != "george-orwell" { + t.Fatalf("expected George Orwell, got %#v", hits) + } + }) + + t.Run("by name case insensitive", func(t *testing.T) { + for _, name := range []string{"orwell", "ORWELL", "oRwElL"} { + res, err := idx.SearchAuthors(index.AuthorSearchFields{Name: name}, 1, 10) + if err != nil { + t.Fatal(err) + } + hits := res.Hits() + if res.TotalHits() != 1 || hits[0].Slug != "george-orwell" { + t.Fatalf("search %q: expected George Orwell, got %#v", name, hits) + } + } + }) + + t.Run("by name unaccented and spaced", func(t *testing.T) { + for _, name := range []string{"perez reverte", "Pérez-Reverte", "perez-reverte", "PEREZ REVERTE"} { + res, err := idx.SearchAuthors(index.AuthorSearchFields{Name: name}, 1, 10) + if err != nil { + t.Fatal(err) + } + hits := res.Hits() + if res.TotalHits() != 1 || hits[0].Slug != "arturo-perez-reverte" { + t.Fatalf("search %q: expected Arturo Pérez-Reverte, got %#v", name, hits) + } + } + }) + + t.Run("by gender", func(t *testing.T) { + female := float64(wikidata.GenderFemale) + res, err := idx.SearchAuthors(index.AuthorSearchFields{Gender: &female}, 1, 10) + if err != nil { + t.Fatal(err) + } + hits := res.Hits() + if res.TotalHits() != 1 || hits[0].Slug != "jane-austen" { + t.Fatalf("expected Jane Austen, got %#v", hits) + } + }) + + t.Run("by birth date range", func(t *testing.T) { + from, _ := date.ParseISO("1900-01-01") + to, _ := date.ParseISO("1920-12-31") + res, err := idx.SearchAuthors(index.AuthorSearchFields{BirthDateFrom: from, BirthDateTo: to}, 1, 10) + if err != nil { + t.Fatal(err) + } + hits := res.Hits() + if res.TotalHits() != 1 || hits[0].Slug != "george-orwell" { + t.Fatalf("expected George Orwell, got %#v", hits) + } + }) + + t.Run("by death date range", func(t *testing.T) { + from, _ := date.ParseISO("1810-01-01") + to, _ := date.ParseISO("1820-12-31") + res, err := idx.SearchAuthors(index.AuthorSearchFields{DeathDateFrom: from, DeathDateTo: to}, 1, 10) + if err != nil { + t.Fatal(err) + } + hits := res.Hits() + if res.TotalHits() != 1 || hits[0].Slug != "jane-austen" { + t.Fatalf("expected Jane Austen, got %#v", hits) + } + }) + + t.Run("by death date to excludes living authors", func(t *testing.T) { + to, _ := date.ParseISO("2020-12-31") + res, err := idx.SearchAuthors(index.AuthorSearchFields{DeathDateTo: to}, 1, 10) + if err != nil { + t.Fatal(err) + } + for _, hit := range res.Hits() { + if hit.Slug == "living-author" { + t.Fatalf("living author should not match death date until filter, got %#v", res.Hits()) + } + } + }) + + t.Run("by document count", func(t *testing.T) { + docs := []index.Document{ + { + ID: "doc-1", + Slug: "doc-1", + AuthorsSlugs: []string{"george-orwell"}, + Metadata: metadata.Metadata{Title: "1984", Authors: []string{"George Orwell"}}, + }, + { + ID: "doc-2", + Slug: "doc-2", + AuthorsSlugs: []string{"george-orwell"}, + Metadata: metadata.Metadata{Title: "Animal Farm", Authors: []string{"George Orwell"}}, + }, + { + ID: "doc-3", + Slug: "doc-3", + AuthorsSlugs: []string{"jane-austen"}, + Metadata: metadata.Metadata{Title: "Pride", Authors: []string{"Jane Austen"}}, + }, + } + for _, doc := range docs { + if err := documentsIndexMem.Index(doc.ID, doc); err != nil { + t.Fatal(err) + } + } + + moreFirst, err := idx.SearchAuthors(index.AuthorSearchFields{SortBy: []string{"-DocumentCount"}}, 1, 10) + if err != nil { + t.Fatal(err) + } + moreHits := moreFirst.Hits() + if len(moreHits) != 4 || + moreHits[0].Slug != "george-orwell" || + moreHits[1].Slug != "jane-austen" || + moreHits[2].Slug != "arturo-perez-reverte" || + moreHits[3].Slug != "living-author" { + t.Fatalf("expected authors sorted by most documents first, got %#v", moreHits) + } + + fewerFirst, err := idx.SearchAuthors(index.AuthorSearchFields{SortBy: []string{"DocumentCount"}}, 1, 10) + if err != nil { + t.Fatal(err) + } + fewerHits := fewerFirst.Hits() + if len(fewerHits) != 4 || + fewerHits[0].Slug != "arturo-perez-reverte" || + fewerHits[1].Slug != "living-author" || + fewerHits[2].Slug != "jane-austen" || + fewerHits[3].Slug != "george-orwell" { + t.Fatalf("expected authors sorted by fewest documents first, got %#v", fewerHits) + } + }) +} + +func mustParseTime(value string) time.Time { + t, _ := time.Parse(time.RFC3339, value) + return t +} diff --git a/internal/index/authors_enrich.go b/internal/index/authors_enrich.go new file mode 100644 index 00000000..abe2587f --- /dev/null +++ b/internal/index/authors_enrich.go @@ -0,0 +1,150 @@ +package index + +import ( + "log" + "time" + + "github.com/blevesearch/bleve/v2" + datasourcemodel "github.com/svera/coreander/v5/internal/datasource/model" +) + +const ( + AuthorEnrichRequestsPerMinute = 200 + DefaultAuthorEnrichInterval = time.Minute / AuthorEnrichRequestsPerMinute +) + +// AuthorDataSource retrieves author metadata from an external source such as Wikidata. +type AuthorDataSource interface { + SearchAuthor(name string, languages []string) (datasourcemodel.Author, error) + SearchEntityIDs(name string) ([]string, error) + RetrieveAuthor(ids []string, languages []string) (datasourcemodel.Author, error) + RetrieveAuthors(candidates map[string][]string, languages []string, batchInterval time.Duration) (map[string]datasourcemodel.Author, error) +} + +// CombineWithDataSource merges external author metadata into an indexed author. +func CombineWithDataSource(author *Author, authorDataSource datasourcemodel.Author, supportedLanguages []string) { + author.DataSourceID = authorDataSource.SourceID() + author.BirthName = authorDataSource.BirthName() + author.RetrievedOn = authorDataSource.RetrievedOn() + author.WikipediaLink = make(map[string]string) + author.InstanceOf = authorDataSource.InstanceOf() + author.Description = make(map[string]string) + author.DateOfBirth = authorDataSource.DateOfBirth() + author.DateOfDeath = authorDataSource.DateOfDeath() + author.Website = authorDataSource.Website() + author.DataSourceImage = authorDataSource.Image() + author.Gender = authorDataSource.Gender() + author.Pseudonyms = make([]string, 0, len(authorDataSource.Pseudonyms())) + + for _, pseudonym := range authorDataSource.Pseudonyms() { + if pseudonym != author.Name { + author.Pseudonyms = append(author.Pseudonyms, pseudonym) + } + } + + for _, lang := range supportedLanguages { + author.WikipediaLink[lang] = authorDataSource.WikipediaLink(lang) + author.Description[lang] = authorDataSource.Description(lang) + } +} + +// AuthorsWithoutInfo returns indexed authors that have not been enriched from an external source yet. +func (b *BleveIndexer) AuthorsWithoutInfo() ([]Author, error) { + count, err := b.authorsIdx.DocCount() + if err != nil { + return nil, err + } + if count == 0 { + return nil, nil + } + + searchReq := bleve.NewSearchRequest(bleve.NewMatchAllQuery()) + searchReq.Fields = []string{"*"} + searchReq.Size = int(count) + + searchResult, err := b.authorsIdx.Search(searchReq) + if err != nil { + return nil, err + } + + authors := make([]Author, 0, len(searchResult.Hits)) + for _, hit := range searchResult.Hits { + author := hydrateAuthor(hit) + if author.RetrievedOn.IsZero() { + authors = append(authors, author) + } + } + return authors, nil +} + +// EnrichAuthorsFromDataSource fetches metadata for authors missing external info and updates the index. +// Name searches are throttled to at most one lookup per interval; entity details are fetched in batches. +func (b *BleveIndexer) EnrichAuthorsFromDataSource(dataSource AuthorDataSource, supportedLanguages []string, interval time.Duration) error { + if interval <= 0 { + interval = DefaultAuthorEnrichInterval + } + + authors, err := b.AuthorsWithoutInfo() + if err != nil { + return err + } + if len(authors) == 0 { + return nil + } + + log.Printf("Enriching %d authors from Wikidata", len(authors)) + + b.beginAuthorEnrichment(len(authors) * 2) + defer b.endAuthorEnrichment() + + ticker := time.NewTicker(interval) + defer ticker.Stop() + + candidates := make(map[string][]string, len(authors)) + authorsBySlug := make(map[string]Author, len(authors)) + + searchIndex := 0 + for _, author := range authors { + authorsBySlug[author.Slug] = author + if author.DataSourceID != "" { + candidates[author.Slug] = []string{author.DataSourceID} + b.recordAuthorEnrichmentProgress() + continue + } + + if searchIndex > 0 { + <-ticker.C + } + searchIndex++ + + ids, err := dataSource.SearchEntityIDs(author.Name) + if err != nil { + log.Printf("Error searching author %s on Wikidata: %s", author.Name, err) + candidates[author.Slug] = nil + } else { + candidates[author.Slug] = ids + } + b.recordAuthorEnrichmentProgress() + } + + enriched, err := dataSource.RetrieveAuthors(candidates, supportedLanguages, interval) + if err != nil { + return err + } + + for slug, author := range authorsBySlug { + if authorDataSource, ok := enriched[slug]; ok { + CombineWithDataSource(&author, authorDataSource, supportedLanguages) + } else { + author.RetrievedOn = time.Now().UTC() + } + + if err := b.IndexAuthor(author); err != nil { + log.Printf("Error indexing enriched author %s: %s", author.Name, err) + } + b.recordAuthorEnrichmentProgress() + } + + log.Printf("Author enrichment finished") + return nil +} diff --git a/internal/index/authors_enrich_test.go b/internal/index/authors_enrich_test.go new file mode 100644 index 00000000..d79c5542 --- /dev/null +++ b/internal/index/authors_enrich_test.go @@ -0,0 +1,247 @@ +package index_test + +import ( + "errors" + "sync" + "testing" + "time" + + "github.com/blevesearch/bleve/v2" + datasourcemodel "github.com/svera/coreander/v5/internal/datasource/model" + "github.com/svera/coreander/v5/internal/index" + "github.com/svera/coreander/v5/internal/precisiondate" +) + +type mockAuthorDataSource struct { + byName map[string]datasourcemodel.Author + bySlug map[string]datasourcemodel.Author + entityIDs map[string][]string + calls int + searchCalls int + retrieveCalls int + delay time.Duration + errName string +} + +func (m *mockAuthorDataSource) SearchAuthor(name string, _ []string) (datasourcemodel.Author, error) { + m.calls++ + m.searchCalls++ + if m.delay > 0 { + time.Sleep(m.delay) + } + if m.errName == name { + return nil, errMockLookup + } + return m.byName[name], nil +} + +var errMockLookup = errors.New("mock lookup failed") + +func (m *mockAuthorDataSource) SearchEntityIDs(name string) ([]string, error) { + m.calls++ + m.searchCalls++ + if m.delay > 0 { + time.Sleep(m.delay) + } + if m.errName == name { + return nil, errMockLookup + } + if m.entityIDs != nil { + if ids, ok := m.entityIDs[name]; ok { + return ids, nil + } + } + if author, ok := m.byName[name]; ok && author != nil { + return []string{author.SourceID()}, nil + } + return nil, nil +} + +func (m *mockAuthorDataSource) RetrieveAuthor(_ []string, _ []string) (datasourcemodel.Author, error) { + m.calls++ + m.retrieveCalls++ + return nil, nil +} + +func (m *mockAuthorDataSource) RetrieveAuthors(candidates map[string][]string, _ []string, _ time.Duration) (map[string]datasourcemodel.Author, error) { + m.calls++ + m.retrieveCalls++ + results := make(map[string]datasourcemodel.Author, len(candidates)) + for slug := range candidates { + if author, ok := m.bySlug[slug]; ok { + results[slug] = author + } + } + return results, nil +} + +type stubAuthor struct { + sourceID string +} + +func (a stubAuthor) BirthName() string { return "" } +func (a stubAuthor) Description(string) string { return "A writer" } +func (a stubAuthor) InstanceOf() float64 { return 1 } +func (a stubAuthor) Gender() float64 { return 0 } +func (a stubAuthor) DateOfBirth() precisiondate.PrecisionDate { return precisiondate.PrecisionDate{} } +func (a stubAuthor) DateOfDeath() precisiondate.PrecisionDate { return precisiondate.PrecisionDate{} } +func (a stubAuthor) Image() string { return "" } +func (a stubAuthor) Website() string { return "" } +func (a stubAuthor) WikipediaLink(string) string { return "" } +func (a stubAuthor) SourceID() string { return a.sourceID } +func (a stubAuthor) RetrievedOn() time.Time { return time.Now().UTC() } +func (a stubAuthor) Pseudonyms() []string { return nil } + +func TestAuthorsWithoutInfo(t *testing.T) { + authorsIndexMem, err := bleve.NewMemOnly(index.CreateAuthorsMapping()) + if err != nil { + t.Fatal(err) + } + documentsIndexMem, err := bleve.NewMemOnly(index.CreateDocumentsMapping()) + if err != nil { + t.Fatal(err) + } + + idx := index.NewBleve(documentsIndexMem, authorsIndexMem, nil, "", nil, index.Config{}) + + if err := idx.IndexAuthor(index.Author{Slug: "enriched", Name: "Enriched Author", RetrievedOn: time.Now().UTC()}); err != nil { + t.Fatal(err) + } + if err := idx.IndexAuthor(index.Author{Slug: "pending", Name: "Pending Author"}); err != nil { + t.Fatal(err) + } + + authors, err := idx.AuthorsWithoutInfo() + if err != nil { + t.Fatal(err) + } + if len(authors) != 1 { + t.Fatalf("expected 1 author without info, got %d", len(authors)) + } + if authors[0].Slug != "pending" { + t.Fatalf("expected pending author, got %q", authors[0].Slug) + } +} + +func TestEnrichAuthorsFromDataSource(t *testing.T) { + authorsIndexMem, err := bleve.NewMemOnly(index.CreateAuthorsMapping()) + if err != nil { + t.Fatal(err) + } + documentsIndexMem, err := bleve.NewMemOnly(index.CreateDocumentsMapping()) + if err != nil { + t.Fatal(err) + } + + idx := index.NewBleve(documentsIndexMem, authorsIndexMem, nil, "", nil, index.Config{}) + + if err := idx.IndexAuthor(index.Author{Slug: "found", Name: "Found Author"}); err != nil { + t.Fatal(err) + } + if err := idx.IndexAuthor(index.Author{Slug: "missing", Name: "Missing Author"}); err != nil { + t.Fatal(err) + } + if err := idx.IndexAuthor(index.Author{Slug: "done", Name: "Done Author", RetrievedOn: time.Now().UTC()}); err != nil { + t.Fatal(err) + } + + dataSource := &mockAuthorDataSource{ + bySlug: map[string]datasourcemodel.Author{ + "found": stubAuthor{sourceID: "Q1"}, + }, + entityIDs: map[string][]string{ + "Found Author": {"Q1"}, + }, + } + + start := time.Now() + if err := idx.EnrichAuthorsFromDataSource(dataSource, []string{"en"}, 50*time.Millisecond); err != nil { + t.Fatal(err) + } + elapsed := time.Since(start) + + if dataSource.searchCalls != 2 { + t.Fatalf("expected 2 Wikidata name searches, got %d", dataSource.searchCalls) + } + if dataSource.retrieveCalls != 1 { + t.Fatalf("expected 1 batched Wikidata entity fetch, got %d", dataSource.retrieveCalls) + } + if elapsed < 50*time.Millisecond { + t.Fatalf("expected throttling between lookups, took %s", elapsed) + } + + found, err := idx.Author("found", "en") + if err != nil { + t.Fatal(err) + } + if found.DataSourceID != "Q1" { + t.Fatalf("expected enriched author Q1, got %q", found.DataSourceID) + } + + missing, err := idx.Author("missing", "en") + if err != nil { + t.Fatal(err) + } + if missing.RetrievedOn.IsZero() { + t.Fatal("expected missing author to be marked as processed") + } + if missing.DataSourceID != "" { + t.Fatalf("expected missing author to have no data source id, got %q", missing.DataSourceID) + } + + remaining, err := idx.AuthorsWithoutInfo() + if err != nil { + t.Fatal(err) + } + if len(remaining) != 0 { + t.Fatalf("expected no authors left to enrich, got %d", len(remaining)) + } +} + +func TestAuthorEnrichmentProgress(t *testing.T) { + authorsIndexMem, err := bleve.NewMemOnly(index.CreateAuthorsMapping()) + if err != nil { + t.Fatal(err) + } + documentsIndexMem, err := bleve.NewMemOnly(index.CreateDocumentsMapping()) + if err != nil { + t.Fatal(err) + } + + idx := index.NewBleve(documentsIndexMem, authorsIndexMem, nil, "", nil, index.Config{}) + + if err := idx.IndexAuthor(index.Author{Slug: "one", Name: "Author One"}); err != nil { + t.Fatal(err) + } + if err := idx.IndexAuthor(index.Author{Slug: "two", Name: "Author Two"}); err != nil { + t.Fatal(err) + } + + dataSource := &mockAuthorDataSource{delay: 40 * time.Millisecond} + + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + _ = idx.EnrichAuthorsFromDataSource(dataSource, []string{"en"}, time.Millisecond) + }() + + deadline := time.Now().Add(2 * time.Second) + sawProgress := false + for time.Now().Before(deadline) { + p, err := idx.IndexingProgress() + if err != nil { + t.Fatal(err) + } + if p.InProgress && p.Percentage > 0 && p.Kind == index.ProgressAuthors { + sawProgress = true + break + } + time.Sleep(10 * time.Millisecond) + } + wg.Wait() + + if !sawProgress { + t.Fatal("expected IndexingProgress to report author enrichment percentage > 0") + } +} diff --git a/internal/index/bleve.go b/internal/index/bleve.go index 370a21b5..a516dc0a 100644 --- a/internal/index/bleve.go +++ b/internal/index/bleve.go @@ -32,7 +32,7 @@ const DocumentVersion = "v12" // AuthorVersion identifies the mapping used for indexing authors. Any changes in the mapping requires an increase // of version, to signal that a new index needs to be created. -const AuthorVersion = "1" +const AuthorVersion = "2" // Metadata fields var ( @@ -64,16 +64,19 @@ type Config struct { } type BleveIndexer struct { - fs afero.Fs - documentsIdx bleve.Index // Documents index - authorsIdx bleve.Index // Authors index - libraryPath string - reader map[string]metadata.Reader - indexStartNanos atomic.Int64 - indexedEntries atomic.Uint64 - indexTotalEntries atomic.Uint64 - illustratedMinAmount int // minimum number of illustrations (excl. cover) for a document to be considered illustrated - illustratedMinSize float64 // minimum size in megapixels for an image to count as an illustration + fs afero.Fs + documentsIdx bleve.Index // Documents index + authorsIdx bleve.Index // Authors index + libraryPath string + reader map[string]metadata.Reader + indexStartNanos atomic.Int64 + indexedEntries atomic.Uint64 + indexTotalEntries atomic.Uint64 + authorEnrichStartNanos atomic.Int64 + authorEnrichProcessed atomic.Uint64 + authorEnrichTotalEntries atomic.Uint64 + illustratedMinAmount int // minimum number of illustrations (excl. cover) for a document to be considered illustrated + illustratedMinSize float64 // minimum size in megapixels for an image to count as an illustration } // NewBleve creates a new BleveIndexer instance using the passed parameters @@ -197,16 +200,35 @@ func CreateDocumentsMapping() mapping.IndexMapping { func CreateAuthorsMapping() mapping.IndexMapping { indexMapping := bleve.NewIndexMapping() + err := indexMapping.AddCustomAnalyzer(defaultAnalyzer, + map[string]any{ + "type": custom.Name, + "char_filters": []string{ + asciifolding.Name, + }, + "tokenizer": unicode.Name, + "token_filters": []string{ + lowercase.Name, + }, + }) + if err != nil { + log.Fatal(err) + } + keywordFieldMapping := bleve.NewKeywordFieldMapping() keywordFieldMappingNotIndexable := bleve.NewKeywordFieldMapping() keywordFieldMappingNotIndexable.Index = false + simpleTextFieldMapping := bleve.NewTextFieldMapping() + simpleTextFieldMapping.Analyzer = defaultAnalyzer + numericFieldMapping := bleve.NewNumericFieldMapping() dateTimeFieldMapping := bleve.NewDateTimeFieldMapping() + indexMapping.DefaultMapping.DefaultAnalyzer = defaultAnalyzer indexMapping.DefaultMapping.AddFieldMappingsAt("Slug", keywordFieldMapping) - indexMapping.DefaultMapping.AddFieldMappingsAt("Name", keywordFieldMapping) - indexMapping.DefaultMapping.AddFieldMappingsAt("BirthName", keywordFieldMapping) + indexMapping.DefaultMapping.AddFieldMappingsAt("Name", simpleTextFieldMapping) + indexMapping.DefaultMapping.AddFieldMappingsAt("BirthName", simpleTextFieldMapping) indexMapping.DefaultMapping.AddFieldMappingsAt("RetrievedOn", dateTimeFieldMapping) indexMapping.DefaultMapping.AddFieldMappingsAt("DataSourceID", keywordFieldMappingNotIndexable) indexMapping.DefaultMapping.AddFieldMappingsAt("DataSourceImage", keywordFieldMappingNotIndexable) diff --git a/internal/index/bleve_read.go b/internal/index/bleve_read.go index 15645d47..25c5b53b 100644 --- a/internal/index/bleve_read.go +++ b/internal/index/bleve_read.go @@ -24,12 +24,17 @@ import ( ) func (b *BleveIndexer) IndexingProgress() (Progress, error) { - if b.indexStartNanos.Load() == 0 { - return Progress{}, nil + if b.indexStartNanos.Load() != 0 { + return b.progressFrom(ProgressDocuments, b.indexStartNanos.Load(), b.indexedEntries.Load(), b.indexTotalEntries.Load()), nil } - processed := b.indexedEntries.Load() - total := b.indexTotalEntries.Load() - progress := Progress{InProgress: true} + if b.authorEnrichStartNanos.Load() != 0 { + return b.progressFrom(ProgressAuthors, b.authorEnrichStartNanos.Load(), b.authorEnrichProcessed.Load(), b.authorEnrichTotalEntries.Load()), nil + } + return Progress{}, nil +} + +func (b *BleveIndexer) progressFrom(kind ProgressKind, startNanos int64, processed, total uint64) Progress { + progress := Progress{Kind: kind, InProgress: true} if total > 0 { progress.Percentage = math.Round(100 * float64(processed) / float64(total)) if progress.Percentage > 100 { @@ -37,10 +42,10 @@ func (b *BleveIndexer) IndexingProgress() (Progress, error) { } } if processed > 0 && processed < total { - elapsed := float64(time.Now().UnixNano()) - float64(b.indexStartNanos.Load()) + elapsed := float64(time.Now().UnixNano()) - float64(startNanos) progress.RemainingTime = time.Duration(elapsed * float64(total-processed) / float64(processed)) } - return progress, nil + return progress } func (b *BleveIndexer) beginIndexing() { @@ -55,6 +60,22 @@ func (b *BleveIndexer) endIndexing() { b.indexTotalEntries.Store(0) } +func (b *BleveIndexer) beginAuthorEnrichment(total int) { + b.authorEnrichStartNanos.Store(time.Now().UnixNano()) + b.authorEnrichProcessed.Store(0) + b.authorEnrichTotalEntries.Store(uint64(total)) +} + +func (b *BleveIndexer) endAuthorEnrichment() { + b.authorEnrichStartNanos.Store(0) + b.authorEnrichProcessed.Store(0) + b.authorEnrichTotalEntries.Store(0) +} + +func (b *BleveIndexer) recordAuthorEnrichmentProgress() { + b.authorEnrichProcessed.Add(1) +} + func countFiles(dir string, fileSystem afero.Fs) (float64, error) { var total float64 @@ -160,20 +181,7 @@ func (b *BleveIndexer) addFilters(searchFields SearchFields, filtersQuery *query filtersQuery.AddQuery(subjectQueries) } } - if searchFields.PubDateFrom != 0 || searchFields.PubDateTo != 0 { - minDate := float64(searchFields.PubDateFrom) - maxDate := float64(searchFields.PubDateTo) - - q := bleve.NewNumericRangeQuery(nil, nil) - if minDate != 0 { - q.Min = &minDate - } - if maxDate != 0 { - q.Max = &maxDate - } - q.SetField("Publication.Date") - filtersQuery.AddQuery(q) - } + addDateRangeFilter(filtersQuery, "Publication.Date", searchFields.PubDateFrom, searchFields.PubDateTo) if searchFields.EstReadTimeFrom > 0 || searchFields.EstReadTimeTo > 0 { q := bleve.NewNumericRangeQuery(nil, nil) if searchFields.EstReadTimeFrom > 0 { @@ -305,6 +313,11 @@ func (b *BleveIndexer) Count() (uint64, error) { return searchResult.Total, nil } +// AuthorsCount returns the number of indexed authors. +func (b *BleveIndexer) AuthorsCount() (uint64, error) { + return b.authorsIdx.DocCount() +} + func (b *BleveIndexer) Document(slug string) (Document, error) { query := bleve.NewTermQuery(slug) query.SetField("Slug") @@ -322,6 +335,21 @@ func (b *BleveIndexer) Document(slug string) (Document, error) { return hydrateDocument(searchResult.Hits[0]), nil } +func (b *BleveIndexer) documentByIndexID(id string) (Document, error) { + query := bleve.NewDocIDQuery([]string{id}) + searchOptions := bleve.NewSearchRequest(query) + searchOptions.Fields = []string{"*"} + searchResult, err := b.documentsIdx.Search(searchOptions) + if err != nil { + return Document{}, err + } + if searchResult.Total == 0 { + return Document{}, nil + } + + return hydrateDocument(searchResult.Hits[0]), nil +} + // IndexedFile holds the bytes and metadata for a document download. type IndexedFile struct { Document Document @@ -557,14 +585,33 @@ func (b *BleveIndexer) Subjects() (map[string][]string, error) { } func (b *BleveIndexer) SearchByAuthor(searchFields SearchFields, page, resultsPerPage int) (result.Paginated[[]Document], error) { - slug := searchFields.Keywords - byAuthor := bleve.NewTermQuery(slug) + return b.runPaginatedQuery(documentQueryByAuthorSlug(searchFields.Keywords), page, resultsPerPage, searchFields.SortBy) +} + +func documentQueryByAuthorSlug(authorSlug string) query.Query { + byAuthor := bleve.NewTermQuery(authorSlug) byAuthor.SetField("AuthorsSlugs") - byIllustrator := bleve.NewTermQuery(slug) + byIllustrator := bleve.NewTermQuery(authorSlug) byIllustrator.SetField("IllustratorsSlugs") - dq := bleve.NewDisjunctionQuery(byAuthor, byIllustrator) + return bleve.NewDisjunctionQuery(byAuthor, byIllustrator) +} - return b.runPaginatedQuery(dq, page, resultsPerPage, searchFields.SortBy) +// DocumentCountsByAuthorSlugs returns document counts keyed by author slug. +func (b *BleveIndexer) DocumentCountsByAuthorSlugs(slugs []string) (map[string]uint64, error) { + counts := make(map[string]uint64, len(slugs)) + for _, authorSlug := range slugs { + if authorSlug == "" { + continue + } + searchRequest := bleve.NewSearchRequest(documentQueryByAuthorSlug(authorSlug)) + searchRequest.Size = 0 + searchResult, err := b.documentsIdx.Search(searchRequest) + if err != nil { + return nil, err + } + counts[authorSlug] = searchResult.Total + } + return counts, nil } func (b *BleveIndexer) Author(slug, lang string) (Author, error) { diff --git a/internal/index/bleve_write.go b/internal/index/bleve_write.go index e731f05b..5641fb6e 100644 --- a/internal/index/bleve_write.go +++ b/internal/index/bleve_write.go @@ -12,6 +12,7 @@ import ( "sync" "time" + "github.com/blevesearch/bleve/v2" index "github.com/blevesearch/bleve_index_api" "github.com/gosimple/slug" "github.com/spf13/afero" @@ -80,12 +81,15 @@ func (b *BleveIndexer) indexFile(file string) (string, error) { // removeFile removes a file from the index func (b *BleveIndexer) removeFile(file string) error { - file = strings.Replace(file, b.libraryPath, "", 1) - file = strings.TrimPrefix(file, string(filepath.Separator)) - if err := b.documentsIdx.Delete(file); err != nil { + id := b.id(file) + document, err := b.documentByIndexID(id) + if err != nil { return err } - return nil + if document.ID != "" { + return b.deleteDocumentFromIndex(document) + } + return b.documentsIdx.Delete(id) } // DeleteDocument removes the document identified by slug from the index and deletes its file from the filesystem. @@ -97,16 +101,58 @@ func (b *BleveIndexer) DeleteDocument(slug string) error { if document.Slug == "" { return ErrDocumentNotFound } - fullPath := filepath.Join(b.libraryPath, document.ID) - if err := b.removeFile(fullPath); err != nil { + if err := b.deleteDocumentFromIndex(document); err != nil { return err } + fullPath := filepath.Join(b.libraryPath, document.ID) if err := b.fs.Remove(fullPath); err != nil && !os.IsNotExist(err) { log.Printf("error removing file %s: %s\n", fullPath, err.Error()) } return nil } +func (b *BleveIndexer) deleteDocumentFromIndex(document Document) error { + if err := b.documentsIdx.Delete(document.ID); err != nil { + return err + } + return b.removeOrphanAuthors(authorSlugsFromDocument(document)) +} + +func authorSlugsFromDocument(document Document) []string { + seen := make(map[string]struct{}) + slugs := make([]string, 0, len(document.AuthorsSlugs)+len(document.IllustratorsSlugs)) + for _, authorSlug := range append(document.AuthorsSlugs, document.IllustratorsSlugs...) { + if authorSlug == "" { + continue + } + if _, ok := seen[authorSlug]; ok { + continue + } + seen[authorSlug] = struct{}{} + slugs = append(slugs, authorSlug) + } + return slugs +} + +func (b *BleveIndexer) removeOrphanAuthors(slugs []string) error { + if len(slugs) == 0 { + return nil + } + counts, err := b.DocumentCountsByAuthorSlugs(slugs) + if err != nil { + return err + } + for _, authorSlug := range slugs { + if counts[authorSlug] != 0 { + continue + } + if err := b.authorsIdx.Delete(authorSlug); err != nil { + return err + } + } + return nil +} + // AddLibrary scans for documents and adds them to the index in batches of if they // haven't been previously indexed or if is true. // metadataWorkers controls parallel metadata extraction after CLI resolution: 1 is fully sequential; values @@ -193,10 +239,82 @@ func (b *BleveIndexer) AddLibrary(batchSize int, forceIndexing bool, metadataWor } } + if err := b.rebuildAuthorsFromDocumentsIfEmpty(); err != nil { + b.endIndexing() + return err + } + b.endIndexing() return nil } +// rebuildAuthorsFromDocumentsIfEmpty repopulates the authors index from indexed documents when +// the authors index is empty but documents remain indexed (for example after a mapping version +// bump recreates the authors index while the documents index is unchanged). +func (b *BleveIndexer) rebuildAuthorsFromDocumentsIfEmpty() error { + authorCount, err := b.authorsIdx.DocCount() + if err != nil { + return err + } + if authorCount > 0 { + return nil + } + + docCount, err := b.Count() + if err != nil { + return err + } + if docCount == 0 { + return nil + } + + log.Println("Authors index is empty but documents are indexed, rebuilding authors from documents index.") + return b.RebuildAuthorsFromDocuments() +} + +// RebuildAuthorsFromDocuments indexes all authors and illustrators referenced by documents +// in the documents index into the authors index. +func (b *BleveIndexer) RebuildAuthorsFromDocuments() error { + countResult, err := b.documentsIdx.Search(bleve.NewSearchRequestOptions(bleve.NewMatchAllQuery(), 0, 0, false)) + if err != nil { + return err + } + total := int(countResult.Total) + if total == 0 { + return nil + } + + searchOptions := bleve.NewSearchRequestOptions(bleve.NewMatchAllQuery(), total, 0, false) + searchOptions.Fields = []string{"*"} + searchResult, err := b.documentsIdx.Search(searchOptions) + if err != nil { + return err + } + + authorsBatch := b.authorsIdx.NewBatch() + authorsSeen := make(map[string]struct{}) + + for _, hit := range searchResult.Hits { + document := hydrateDocument(hit) + if err := b.indexAuthors(document, authorsBatch.Index, authorsSeen); err != nil { + return err + } + if authorsBatch.Size() >= 100 { + if err := b.authorsIdx.Batch(authorsBatch); err != nil { + return err + } + authorsBatch.Reset() + } + } + + if authorsBatch.Size() > 0 { + if err := b.authorsIdx.Batch(authorsBatch); err != nil { + return err + } + } + return nil +} + func (b *BleveIndexer) collectPendingLibraryPaths(forceIndexing bool) (pending []string, languages []string, err error) { languages = []string{} e := afero.Walk(b.fs, b.libraryPath, func(fullPath string, f os.FileInfo, walkErr error) error { diff --git a/internal/index/date_range_filter.go b/internal/index/date_range_filter.go new file mode 100644 index 00000000..137aa4f2 --- /dev/null +++ b/internal/index/date_range_filter.go @@ -0,0 +1,24 @@ +package index + +import ( + "github.com/blevesearch/bleve/v2" + "github.com/blevesearch/bleve/v2/search/query" + "github.com/rickb777/date/v2" +) + +func addDateRangeFilter(filtersQuery *query.ConjunctionQuery, field string, from, to date.Date) { + if from == 0 && to == 0 { + return + } + minDate := float64(from) + q := bleve.NewNumericRangeQuery(nil, nil) + if from != 0 { + q.Min = &minDate + } + if to != 0 { + maxDate := float64(to) + 1 + q.Max = &maxDate + } + q.SetField(field) + filtersQuery.AddQuery(q) +} diff --git a/internal/index/document_counts_test.go b/internal/index/document_counts_test.go new file mode 100644 index 00000000..17e236a5 --- /dev/null +++ b/internal/index/document_counts_test.go @@ -0,0 +1,62 @@ +package index_test + +import ( + "testing" + + "github.com/blevesearch/bleve/v2" + "github.com/svera/coreander/v5/internal/index" + "github.com/svera/coreander/v5/internal/metadata" +) + +func TestDocumentCountsByAuthorSlugs(t *testing.T) { + documentsIndexMem, err := bleve.NewMemOnly(index.CreateDocumentsMapping()) + if err != nil { + t.Fatal(err) + } + authorsIndexMem, err := bleve.NewMemOnly(index.CreateAuthorsMapping()) + if err != nil { + t.Fatal(err) + } + + idx := index.NewBleve(documentsIndexMem, authorsIndexMem, nil, "", nil, index.Config{}) + + docs := []index.Document{ + { + ID: "doc-1", + Slug: "doc-1", + AuthorsSlugs: []string{"george-orwell"}, + Metadata: metadata.Metadata{Title: "1984", Authors: []string{"George Orwell"}}, + }, + { + ID: "doc-2", + Slug: "doc-2", + AuthorsSlugs: []string{"george-orwell"}, + Metadata: metadata.Metadata{Title: "Animal Farm", Authors: []string{"George Orwell"}}, + }, + { + ID: "doc-3", + Slug: "doc-3", + IllustratorsSlugs: []string{"jane-austen"}, + Metadata: metadata.Metadata{Title: "Illustrated", Illustrators: []string{"Jane Austen"}}, + }, + } + for _, doc := range docs { + if err := documentsIndexMem.Index(doc.ID, doc); err != nil { + t.Fatal(err) + } + } + + counts, err := idx.DocumentCountsByAuthorSlugs([]string{"george-orwell", "jane-austen", "unknown"}) + if err != nil { + t.Fatal(err) + } + if counts["george-orwell"] != 2 { + t.Fatalf("expected 2 documents for George Orwell, got %d", counts["george-orwell"]) + } + if counts["jane-austen"] != 1 { + t.Fatalf("expected 1 document for Jane Austen, got %d", counts["jane-austen"]) + } + if counts["unknown"] != 0 { + t.Fatalf("expected 0 documents for unknown author, got %d", counts["unknown"]) + } +} diff --git a/internal/index/progress.go b/internal/index/progress.go index ab479e03..5865d7d5 100644 --- a/internal/index/progress.go +++ b/internal/index/progress.go @@ -2,7 +2,15 @@ package index import "time" +type ProgressKind string + +const ( + ProgressDocuments ProgressKind = "documents" + ProgressAuthors ProgressKind = "authors" +) + type Progress struct { + Kind ProgressKind InProgress bool RemainingTime time.Duration Percentage float64 diff --git a/internal/index/rebuild_authors_test.go b/internal/index/rebuild_authors_test.go new file mode 100644 index 00000000..fde0549f --- /dev/null +++ b/internal/index/rebuild_authors_test.go @@ -0,0 +1,63 @@ +package index_test + +import ( + "testing" + + "github.com/blevesearch/bleve/v2" + "github.com/svera/coreander/v5/internal/index" + "github.com/svera/coreander/v5/internal/metadata" +) + +func TestRebuildAuthorsFromDocuments(t *testing.T) { + documentsIndexMem, err := bleve.NewMemOnly(index.CreateDocumentsMapping()) + if err != nil { + t.Fatal(err) + } + authorsIndexMem, err := bleve.NewMemOnly(index.CreateAuthorsMapping()) + if err != nil { + t.Fatal(err) + } + + idx := index.NewBleve(documentsIndexMem, authorsIndexMem, nil, "", nil, index.Config{}) + + docs := []index.Document{ + { + ID: "doc-1", + Slug: "doc-1", + AuthorsSlugs: []string{"george-orwell"}, + Metadata: metadata.Metadata{Title: "1984", Authors: []string{"George Orwell"}}, + }, + { + ID: "doc-2", + Slug: "doc-2", + IllustratorsSlugs: []string{"jane-austen"}, + Metadata: metadata.Metadata{Title: "Illustrated", Illustrators: []string{"Jane Austen"}}, + }, + } + for _, doc := range docs { + if err := documentsIndexMem.Index(doc.ID, doc); err != nil { + t.Fatal(err) + } + } + + if count, err := authorsIndexMem.DocCount(); err != nil { + t.Fatal(err) + } else if count != 0 { + t.Fatalf("expected empty authors index, got %d", count) + } + + if err := idx.RebuildAuthorsFromDocuments(); err != nil { + t.Fatal(err) + } + + if count, err := authorsIndexMem.DocCount(); err != nil { + t.Fatal(err) + } else if count != 2 { + t.Fatalf("expected 2 authors after rebuild, got %d", count) + } + + author, err := idx.Author("george-orwell", "en") + if err != nil || author.Name != "George Orwell" { + t.Fatalf("expected George Orwell, got %#v err=%v", author, err) + } +} diff --git a/internal/result/paginated.go b/internal/result/paginated.go index be091978..7a09ab0e 100644 --- a/internal/result/paginated.go +++ b/internal/result/paginated.go @@ -22,6 +22,33 @@ func NewPaginated[T any](maxResultsPerPage, page, totalHits int, hits T) Paginat } } +// Paginate wraps hits in pagination metadata. When len(hits) equals totalHits, hits is treated +// as the full result set and sliced for the requested page. Otherwise hits is assumed to already +// contain the page window (for example from a database or Bleve query). +func Paginate[T any](maxResultsPerPage, page, totalHits int, hits []T) Paginated[[]T] { + if page < 1 { + page = 1 + } + if totalHits == 0 { + return NewPaginated(maxResultsPerPage, page, 0, []T{}) + } + + start := (page - 1) * maxResultsPerPage + if start >= totalHits { + return NewPaginated(maxResultsPerPage, page, totalHits, []T{}) + } + + if len(hits) == totalHits { + end := start + maxResultsPerPage + if end > totalHits { + end = totalHits + } + return NewPaginated(maxResultsPerPage, page, totalHits, hits[start:end]) + } + + return NewPaginated(maxResultsPerPage, page, totalHits, hits) +} + func (P Paginated[T]) MaxResultsPerPage() int { return P.maxResultsPerPage } diff --git a/internal/result/paginated_test.go b/internal/result/paginated_test.go new file mode 100644 index 00000000..5b67b3cb --- /dev/null +++ b/internal/result/paginated_test.go @@ -0,0 +1,37 @@ +package result_test + +import ( + "testing" + + "github.com/svera/coreander/v5/internal/result" +) + +func TestPaginateFullList(t *testing.T) { + items := []int{1, 2, 3, 4, 5} + + page := result.Paginate(2, 2, len(items), items) + if got := page.Hits(); len(got) != 2 || got[0] != 3 || got[1] != 4 { + t.Fatalf("expected page [3 4], got %#v", got) + } + if page.TotalHits() != 5 || page.Page() != 2 { + t.Fatalf("unexpected metadata: total=%d page=%d", page.TotalHits(), page.Page()) + } +} + +func TestPaginatePrePaginatedWindow(t *testing.T) { + pageWindow := []string{"c", "d"} + + page := result.Paginate(2, 2, 5, pageWindow) + if got := page.Hits(); len(got) != 2 || got[0] != "c" || got[1] != "d" { + t.Fatalf("expected pre-paginated window unchanged, got %#v", got) + } +} + +func TestPaginateEmptyPage(t *testing.T) { + items := []int{1, 2, 3} + + page := result.Paginate(2, 3, len(items), items) + if len(page.Hits()) != 0 { + t.Fatalf("expected empty page, got %#v", page.Hits()) + } +} diff --git a/internal/webserver/controller.go b/internal/webserver/controller.go index dad27cfc..9e3551bc 100644 --- a/internal/webserver/controller.go +++ b/internal/webserver/controller.go @@ -10,6 +10,7 @@ import ( "github.com/svera/coreander/v5/internal/webserver/controller/document" "github.com/svera/coreander/v5/internal/webserver/controller/highlight" "github.com/svera/coreander/v5/internal/webserver/controller/home" + "github.com/svera/coreander/v5/internal/webserver/controller/search" "github.com/svera/coreander/v5/internal/webserver/controller/series" "github.com/svera/coreander/v5/internal/webserver/controller/user" "github.com/svera/coreander/v5/internal/webserver/model" @@ -24,6 +25,7 @@ type Controllers struct { Documents *document.Controller Home *home.Controller Authors *author.Controller + Search *search.Controller Series *series.Controller } @@ -85,6 +87,10 @@ func SetupControllers(cfg Config, db *gorm.DB, metadataReaders map[string]metada WordsPerMinute: cfg.WordsPerMinute, } + searchCfg := search.Config{ + WordsPerMinute: cfg.WordsPerMinute, + } + homeCfg := home.Config{ LibraryPath: cfg.LibraryPath, CoverMaxWidth: cfg.CoverMaxWidth, @@ -99,5 +105,6 @@ func SetupControllers(cfg Config, db *gorm.DB, metadataReaders map[string]metada Documents: document.NewController(highlightsRepository, usersRepository, readingRepository, sender, idx, metadataReaders, appFs, documentsCfg, translator), Home: home.NewController(highlightsRepository, readingRepository, sender, idx, homeCfg), Authors: author.NewController(highlightsRepository, readingRepository, sender, idx, authorsCfg, dataSource, appFs, imagesFS), + Search: search.NewController(highlightsRepository, readingRepository, sender, idx, searchCfg), Series: series.NewController(highlightsRepository, readingRepository, sender, idx, seriesCfg, appFs)} } diff --git a/internal/webserver/controller/author/controller.go b/internal/webserver/controller/author/controller.go index fcdd60ca..cde9f06a 100644 --- a/internal/webserver/controller/author/controller.go +++ b/internal/webserver/controller/author/controller.go @@ -15,7 +15,9 @@ type Sender interface { // IdxReader defines a set of author reading operations over an index type IdxReader interface { + SearchAuthors(searchFields index.AuthorSearchFields, page, resultsPerPage int) (result.Paginated[[]index.Author], error) SearchByAuthor(searchFields index.SearchFields, page, resultsPerPage int) (result.Paginated[[]index.Document], error) + DocumentCountsByAuthorSlugs(slugs []string) (map[string]uint64, error) Author(slug, lang string) (index.Author, error) IndexAuthor(author index.Author) error Languages() ([]string, error) diff --git a/internal/webserver/controller/author/image.go b/internal/webserver/controller/author/image.go index 56102180..0b367277 100644 --- a/internal/webserver/controller/author/image.go +++ b/internal/webserver/controller/author/image.go @@ -143,7 +143,7 @@ func (a *Controller) readFromDataSource(path string) (image.Image, error) { return nil, fmt.Errorf("failed to fetch image from %s: HTTP %d", path, res.StatusCode) } - img, err := imaging.Decode(res.Body, imaging.Backends(imaging.GO_IMAGE)) + img, err := imaging.Decode(res.Body) if err != nil { return nil, fmt.Errorf("failed to decode image from %s: %w", path, err) } diff --git a/internal/webserver/controller/author/summary.go b/internal/webserver/controller/author/summary.go index b1f7ce88..955ae883 100644 --- a/internal/webserver/controller/author/summary.go +++ b/internal/webserver/controller/author/summary.go @@ -5,7 +5,7 @@ import ( "log" "github.com/gofiber/fiber/v3" - "github.com/svera/coreander/v5/internal/datasource/model" + datasourcemodel "github.com/svera/coreander/v5/internal/datasource/model" "github.com/svera/coreander/v5/internal/index" ) @@ -16,7 +16,7 @@ func (a *Controller) Summary(c fiber.Ctx) error { c.Set("Pragma", "no-cache") c.Set("Expires", "0") var ( - authorDataSource model.Author + authorDataSource datasourcemodel.Author err error ) @@ -68,7 +68,7 @@ func (a *Controller) Summary(c fiber.Ctx) error { return fiber.ErrNotFound } - combineWithDataSource(&author, authorDataSource, supportedLanguages) + index.CombineWithDataSource(&author, authorDataSource, supportedLanguages) if err := a.idx.IndexAuthor(author); err != nil { log.Println(err) @@ -86,32 +86,6 @@ func (a *Controller) Summary(c fiber.Ctx) error { return nil } -func combineWithDataSource(author *index.Author, authorDataSource model.Author, supportedLanguages []string) { - author.DataSourceID = authorDataSource.SourceID() - author.BirthName = authorDataSource.BirthName() - author.RetrievedOn = authorDataSource.RetrievedOn() - author.WikipediaLink = make(map[string]string) - author.InstanceOf = authorDataSource.InstanceOf() - author.Description = make(map[string]string) - author.DateOfBirth = authorDataSource.DateOfBirth() - author.DateOfDeath = authorDataSource.DateOfDeath() - author.Website = authorDataSource.Website() - author.DataSourceImage = authorDataSource.Image() - author.Gender = authorDataSource.Gender() - author.Pseudonyms = make([]string, 0, len(authorDataSource.Pseudonyms())) - - for _, pseudonym := range authorDataSource.Pseudonyms() { - if pseudonym != author.Name { - author.Pseudonyms = append(author.Pseudonyms, pseudonym) - } - } - - for _, lang := range supportedLanguages { - author.WikipediaLink[lang] = authorDataSource.WikipediaLink(lang) - author.Description[lang] = authorDataSource.Description(lang) - } -} - // getImageVersion returns the modification time of the cached image file as a cache-busting version // Returns empty string if file doesn't exist func (a *Controller) getImageVersion(authorSlug string) string { diff --git a/internal/webserver/controller/author/update.go b/internal/webserver/controller/author/update.go index 37bd39b1..01785342 100644 --- a/internal/webserver/controller/author/update.go +++ b/internal/webserver/controller/author/update.go @@ -50,7 +50,7 @@ func (a *Controller) Update(c fiber.Ctx) error { fmt.Println(err) } - combineWithDataSource(&author, authorDataSource, supportedLanguages) + index.CombineWithDataSource(&author, authorDataSource, supportedLanguages) } if err := a.idx.IndexAuthor(author); err != nil { diff --git a/internal/webserver/controller/document/search.go b/internal/webserver/controller/document/search.go deleted file mode 100644 index f027aaed..00000000 --- a/internal/webserver/controller/document/search.go +++ /dev/null @@ -1,150 +0,0 @@ -package document - -import ( - "log" - "strconv" - - "github.com/gofiber/fiber/v3" - "github.com/rickb777/date/v2" - "github.com/svera/coreander/v5/internal/index" - "github.com/svera/coreander/v5/internal/result" - "github.com/svera/coreander/v5/internal/webserver/model" - "github.com/svera/coreander/v5/internal/webserver/view" -) - -func (d *Controller) Search(c fiber.Ctx) error { - var session model.Session - if val, ok := c.Locals("Session").(model.Session); ok { - session = val - } - - if session.WordsPerMinute > 0 { - d.config.WordsPerMinute = session.WordsPerMinute - } - - var documentResults result.Paginated[[]index.Document] - searchFields, err := d.parseSearchQuery(c) - if err != nil { - log.Println(err) - return fiber.ErrBadRequest - } - - page, err := strconv.Atoi(c.Query("page")) - if err != nil { - page = 1 - } - - if documentResults, err = d.idx.Search(searchFields, page, model.ResultsPerPage); err != nil { - log.Println(err) - return fiber.ErrInternalServerError - } - - searchResults := model.AugmentedDocumentsFromDocuments(documentResults) - if session.ID > 0 { - searchResults = d.readingRepository.CompletedPaginatedResult(int(session.ID), searchResults) - searchResults = d.hlRepository.HighlightedPaginatedResult(int(session.ID), searchResults) - } - - templateVars := fiber.Map{ - "SearchFields": searchFields, - "Results": searchResults, - "Paginator": view.Pagination(model.MaxPagesNavigator, searchResults, c.Queries()), - "Title": "Search results", - "DocumentsSearchPage": true, - "EmailFrom": d.sender.From(), - "WordsPerMinute": d.config.WordsPerMinute, - "URL": view.URL(c), - "SortURL": view.BaseURLWithout(c, "sort-by", "page"), - "SortBy": c.Query("sort-by"), - "AdditionalSortOptions": []struct { - Key string - Value string - }{ - {"relevance", "relevance"}, - {"pub-date-older-first", "older"}, - {"pub-date-newer-first", "newer"}, - {"est-read-time-shorter-first", "shorter"}, - {"est-read-time-longer-first", "longer"}, - }, - } - - if c.Get("hx-request") == "true" { - if err = c.Render("partials/docs-list-fragments", templateVars); err != nil { - log.Println(err) - return fiber.ErrInternalServerError - } - return nil - } - - if err = c.Render("document/list", templateVars, "layout"); err != nil { - log.Println(err) - return fiber.ErrInternalServerError - } - - return nil -} - -func (d *Controller) parseSearchQuery(c fiber.Ctx) (index.SearchFields, error) { - searchFields := index.SearchFields{ - Keywords: c.Query("search"), - Language: c.Query("language"), - Subjects: c.Query("subjects"), - SortBy: d.parseSortBy(c), - EstReadTimeFrom: fiber.Query[float64](c, "est-read-time-from", 0), - EstReadTimeTo: fiber.Query[float64](c, "est-read-time-to", 0), - WordsPerMinute: d.config.WordsPerMinute, - IllustratedOnly: c.Query("illustrated-only") == "on" || c.Query("illustrated-only") == "1", - } - - if c.Query("pub-date-from") != "" { - pubDateFrom, err := date.ParseISO(c.Query("pub-date-from")) - if err != nil { - return searchFields, err - } - searchFields.PubDateFrom = pubDateFrom - } - - if c.Query("pub-date-to") != "" { - pubDateTo, err := date.ParseISO(c.Query("pub-date-to")) - if err != nil { - return searchFields, err - } - searchFields.PubDateTo = pubDateTo - } - - if searchFields.PubDateTo != 0 && searchFields.PubDateFrom > searchFields.PubDateTo { - searchFields.PubDateFrom, searchFields.PubDateTo = searchFields.PubDateTo, searchFields.PubDateFrom - } - - if searchFields.EstReadTimeTo != 0 && searchFields.EstReadTimeFrom > searchFields.EstReadTimeTo { - searchFields.EstReadTimeFrom, searchFields.EstReadTimeTo = searchFields.EstReadTimeTo, searchFields.EstReadTimeFrom - } - - return searchFields, nil -} - -func (d *Controller) parseSortBy(c fiber.Ctx) []string { - if c.Query("sort-by") != "" { - switch c.Query("sort-by") { - case "pub-date-older-first": - return []string{"Publication.Date"} - case "pub-date-newer-first": - return []string{"-Publication.Date"} - case "est-read-time-shorter-first": - return []string{"Words"} - case "est-read-time-longer-first": - return []string{"-Words"} - } - } - return []string{"-_score", "Series", "SeriesIndex"} -} - -// Subjects returns all subjects from the index grouped by slug (map[slug][]names), as JSON -func (d *Controller) Subjects(c fiber.Ctx) error { - bySlug, err := d.idx.Subjects() - if err != nil { - log.Println(err) - return fiber.ErrInternalServerError - } - return c.JSON(bySlug) -} diff --git a/internal/webserver/controller/home/controller.go b/internal/webserver/controller/home/controller.go index cb6e8a62..ea5d525e 100644 --- a/internal/webserver/controller/home/controller.go +++ b/internal/webserver/controller/home/controller.go @@ -15,6 +15,7 @@ type Sender interface { type IdxReaderWriter interface { Document(slug string) (index.Document, error) Count() (uint64, error) + AuthorsCount() (uint64, error) LatestDocs(limit int) ([]index.Document, error) Languages() ([]string, error) } diff --git a/internal/webserver/controller/home/index.go b/internal/webserver/controller/home/index.go index 8d478055..c416f572 100644 --- a/internal/webserver/controller/home/index.go +++ b/internal/webserver/controller/home/index.go @@ -2,12 +2,18 @@ package home import ( "log" + "net/url" + "strings" "github.com/gofiber/fiber/v3" "github.com/svera/coreander/v5/internal/webserver/model" ) func (d *Controller) Index(c fiber.Ctx) error { + if query := strings.TrimSpace(c.Query("search")); query != "" { + return c.Redirect().To("/search?type=documents&search=" + url.QueryEscape(query)) + } + var session model.Session if val, ok := c.Locals("Session").(model.Session); ok { session = val @@ -19,6 +25,12 @@ func (d *Controller) Index(c fiber.Ctx) error { return fiber.ErrInternalServerError } + totalAuthorsCount, err := d.idx.AuthorsCount() + if err != nil { + log.Println(err) + return fiber.ErrInternalServerError + } + latestDocsRaw, err := d.idx.LatestDocs(d.config.LatestDocsLimit) if err != nil { log.Println(err) @@ -46,10 +58,11 @@ func (d *Controller) Index(c fiber.Ctx) error { } return c.Render("index", fiber.Map{ - "Count": totalDocumentsCount, - "EmailFrom": d.sender.From(), - "HomeNavbar": true, - "LatestDocs": latestDocs, - "Reading": readingDocs, + "Count": totalDocumentsCount, + "AuthorsCount": totalAuthorsCount, + "EmailFrom": d.sender.From(), + "HomeNavbar": true, + "LatestDocs": latestDocs, + "Reading": readingDocs, }, "layout") } diff --git a/internal/webserver/controller/search/controller.go b/internal/webserver/controller/search/controller.go new file mode 100644 index 00000000..4389054e --- /dev/null +++ b/internal/webserver/controller/search/controller.go @@ -0,0 +1,55 @@ +package search + +import ( + "github.com/svera/coreander/v5/internal/index" + "github.com/svera/coreander/v5/internal/result" + "github.com/svera/coreander/v5/internal/webserver/model" +) + +const ( + TypeDocuments = "documents" + TypeAuthors = "authors" +) + +type Sender interface { + From() string +} + +type IdxReader interface { + Search(searchFields index.SearchFields, page, resultsPerPage int) (result.Paginated[[]index.Document], error) + SearchAuthors(searchFields index.AuthorSearchFields, page, resultsPerPage int) (result.Paginated[[]index.Author], error) + DocumentCountsByAuthorSlugs(slugs []string) (map[string]uint64, error) + Count() (uint64, error) + AuthorsCount() (uint64, error) + Subjects() (map[string][]string, error) +} + +type highlightsRepository interface { + HighlightedPaginatedResult(userID int, results result.Paginated[[]model.AugmentedDocument]) result.Paginated[[]model.AugmentedDocument] +} + +type readingRepository interface { + CompletedPaginatedResult(userID int, results result.Paginated[[]model.AugmentedDocument]) result.Paginated[[]model.AugmentedDocument] +} + +type Config struct { + WordsPerMinute float64 +} + +type Controller struct { + hlRepository highlightsRepository + readingRepository readingRepository + idx IdxReader + sender Sender + config Config +} + +func NewController(hlRepository highlightsRepository, readingRepository readingRepository, sender Sender, idx IdxReader, cfg Config) *Controller { + return &Controller{ + hlRepository: hlRepository, + readingRepository: readingRepository, + idx: idx, + sender: sender, + config: cfg, + } +} diff --git a/internal/webserver/controller/search/parse.go b/internal/webserver/controller/search/parse.go new file mode 100644 index 00000000..a5a8ac15 --- /dev/null +++ b/internal/webserver/controller/search/parse.go @@ -0,0 +1,199 @@ +package search + +import ( + "log" + "strconv" + + "github.com/gofiber/fiber/v3" + "github.com/rickb777/date/v2" + "github.com/svera/coreander/v5/internal/datasource/wikidata" + "github.com/svera/coreander/v5/internal/index" +) + +func parseDocumentSearchQuery(c fiber.Ctx, wordsPerMinute float64) (index.SearchFields, error) { + searchFields := index.SearchFields{ + Keywords: c.Query("search"), + Language: c.Query("language"), + Subjects: c.Query("subjects"), + SortBy: parseDocumentSortBy(c), + EstReadTimeFrom: fiber.Query[float64](c, "est-read-time-from", 0), + EstReadTimeTo: fiber.Query[float64](c, "est-read-time-to", 0), + WordsPerMinute: wordsPerMinute, + IllustratedOnly: c.Query("illustrated-only") == "on" || c.Query("illustrated-only") == "1", + } + + if c.Query("pub-date-from") != "" { + pubDateFrom, err := date.ParseISO(c.Query("pub-date-from")) + if err != nil { + return searchFields, err + } + searchFields.PubDateFrom = pubDateFrom + } + + if c.Query("pub-date-to") != "" { + pubDateTo, err := date.ParseISO(c.Query("pub-date-to")) + if err != nil { + return searchFields, err + } + searchFields.PubDateTo = pubDateTo + } + + if searchFields.PubDateTo != 0 && searchFields.PubDateFrom > searchFields.PubDateTo { + searchFields.PubDateFrom, searchFields.PubDateTo = searchFields.PubDateTo, searchFields.PubDateFrom + } + + if searchFields.EstReadTimeTo != 0 && searchFields.EstReadTimeFrom > searchFields.EstReadTimeTo { + searchFields.EstReadTimeFrom, searchFields.EstReadTimeTo = searchFields.EstReadTimeTo, searchFields.EstReadTimeFrom + } + + return searchFields, nil +} + +func parseAuthorSearchQuery(c fiber.Ctx) (index.AuthorSearchFields, error) { + name := c.Query("name") + if name == "" { + name = c.Query("search") + } + + searchFields := index.AuthorSearchFields{ + Name: name, + SortBy: parseAuthorSortBy(c), + } + + if gender, ok := parseGenderQuery(c.Query("gender")); ok { + searchFields.Gender = &gender + } + + if c.Query("birthdate-from") != "" { + birthDateFrom, err := date.ParseISO(c.Query("birthdate-from")) + if err != nil { + return searchFields, err + } + searchFields.BirthDateFrom = birthDateFrom + } + + if c.Query("birthdate-to") != "" { + birthDateTo, err := date.ParseISO(c.Query("birthdate-to")) + if err != nil { + return searchFields, err + } + searchFields.BirthDateTo = birthDateTo + } + + if c.Query("deathdate-from") != "" { + deathDateFrom, err := date.ParseISO(c.Query("deathdate-from")) + if err != nil { + return searchFields, err + } + searchFields.DeathDateFrom = deathDateFrom + } + + if c.Query("deathdate-to") != "" { + deathDateTo, err := date.ParseISO(c.Query("deathdate-to")) + if err != nil { + return searchFields, err + } + searchFields.DeathDateTo = deathDateTo + } + + if searchFields.BirthDateTo != 0 && searchFields.BirthDateFrom > searchFields.BirthDateTo { + searchFields.BirthDateFrom, searchFields.BirthDateTo = searchFields.BirthDateTo, searchFields.BirthDateFrom + } + if searchFields.DeathDateTo != 0 && searchFields.DeathDateFrom > searchFields.DeathDateTo { + searchFields.DeathDateFrom, searchFields.DeathDateTo = searchFields.DeathDateTo, searchFields.DeathDateFrom + } + + return searchFields, nil +} + +func parseGenderQuery(value string) (float64, bool) { + switch value { + case "male": + return wikidata.GenderMale, true + case "female": + return wikidata.GenderFemale, true + case "intersex": + return wikidata.GenderIntersex, true + case "transgender-female": + return wikidata.GenderTrasgenderFemale, true + case "transgender-male": + return wikidata.GenderTrasgenderMale, true + case "unknown": + return wikidata.GenderUnknown, true + default: + return 0, false + } +} + +func parseDocumentSortBy(c fiber.Ctx) []string { + if c.Query("sort-by") != "" { + switch c.Query("sort-by") { + case "pub-date-older-first": + return []string{"Publication.Date"} + case "pub-date-newer-first": + return []string{"-Publication.Date"} + case "est-read-time-shorter-first": + return []string{"Words"} + case "est-read-time-longer-first": + return []string{"-Words"} + } + } + return []string{"-_score", "Series", "SeriesIndex"} +} + +func parseAuthorSortBy(c fiber.Ctx) []string { + switch c.Query("sort-by") { + case "name-z-a": + return []string{"-Name"} + case "birth-older-first": + return []string{"DateOfBirth.Date"} + case "birth-newer-first": + return []string{"-DateOfBirth.Date"} + case "death-older-first": + return []string{"DateOfDeath.Date"} + case "death-newer-first": + return []string{"-DateOfDeath.Date"} + case "documents-more-first": + return []string{"-DocumentCount"} + case "documents-fewer-first": + return []string{"DocumentCount"} + default: + return []string{"Name"} + } +} + +func searchTypeFromContext(c fiber.Ctx) string { + if val, ok := c.Locals("SearchType").(string); ok && val != "" { + return val + } + if t := c.Query("type"); t == TypeAuthors { + return TypeAuthors + } + return TypeDocuments +} + +func parsePage(c fiber.Ctx) int { + page, err := strconv.Atoi(c.Query("page")) + if err != nil { + return 1 + } + return page +} + +func authorSlugs(authors []index.Author) []string { + slugs := make([]string, len(authors)) + for i, author := range authors { + slugs[i] = author.Slug + } + return slugs +} + +// Subjects returns all subjects from the index grouped by slug, as JSON. +func (s *Controller) Subjects(c fiber.Ctx) error { + bySlug, err := s.idx.Subjects() + if err != nil { + log.Println(err) + return fiber.ErrInternalServerError + } + return c.JSON(bySlug) +} diff --git a/internal/webserver/controller/search/search.go b/internal/webserver/controller/search/search.go new file mode 100644 index 00000000..d7e22e9a --- /dev/null +++ b/internal/webserver/controller/search/search.go @@ -0,0 +1,183 @@ +package search + +import ( + "log" + + "github.com/gofiber/fiber/v3" + "github.com/svera/coreander/v5/internal/index" + "github.com/svera/coreander/v5/internal/result" + "github.com/svera/coreander/v5/internal/webserver/model" + "github.com/svera/coreander/v5/internal/webserver/view" +) + +func (s *Controller) SearchDocuments(c fiber.Ctx) error { + c.Locals("SearchType", TypeDocuments) + return s.Search(c) +} + +func (s *Controller) SearchAuthors(c fiber.Ctx) error { + c.Locals("SearchType", TypeAuthors) + return s.Search(c) +} + +func (s *Controller) Search(c fiber.Ctx) error { + searchType := searchTypeFromContext(c) + + var session model.Session + if val, ok := c.Locals("Session").(model.Session); ok { + session = val + } + + wordsPerMinute := s.config.WordsPerMinute + if session.WordsPerMinute > 0 { + wordsPerMinute = session.WordsPerMinute + } + + page := parsePage(c) + + if searchType == TypeAuthors { + return s.renderAuthorSearch(c, session, page) + } + return s.renderDocumentSearch(c, session, page, wordsPerMinute) +} + +func (s *Controller) renderDocumentSearch(c fiber.Ctx, session model.Session, page int, wordsPerMinute float64) error { + searchFields, err := parseDocumentSearchQuery(c, wordsPerMinute) + if err != nil { + log.Println(err) + return fiber.ErrBadRequest + } + + documentResults, err := s.idx.Search(searchFields, page, model.ResultsPerPage) + if err != nil { + log.Println(err) + return fiber.ErrInternalServerError + } + + searchResults := model.AugmentedDocumentsFromDocuments(documentResults) + if session.ID > 0 { + searchResults = s.readingRepository.CompletedPaginatedResult(int(session.ID), searchResults) + searchResults = s.hlRepository.HighlightedPaginatedResult(int(session.ID), searchResults) + } + + templateVars := s.baseTemplateVars(c, TypeDocuments) + templateVars["SearchFields"] = searchFields + templateVars["DocumentSearchFields"] = searchFields + templateVars["AuthorSearchFields"] = index.AuthorSearchFields{} + templateVars["SearchQuery"] = searchFields.Keywords + templateVars["Results"] = searchResults + templateVars["Title"] = "Search results" + templateVars["WordsPerMinute"] = wordsPerMinute + templateVars["SortBy"] = c.Query("sort-by") + templateVars["AdditionalSortOptions"] = documentSortOptions() + + return s.renderSearch(c, templateVars, "partials/docs-list-fragments") +} + +func (s *Controller) renderAuthorSearch(c fiber.Ctx, session model.Session, page int) error { + searchFields, err := parseAuthorSearchQuery(c) + if err != nil { + log.Println(err) + return fiber.ErrBadRequest + } + + authorResults, err := s.idx.SearchAuthors(searchFields, page, int(model.ResultsPerPage)) + if err != nil { + log.Println(err) + return fiber.ErrInternalServerError + } + + documentCounts := map[string]uint64{} + if slugs := authorSlugs(authorResults.Hits()); len(slugs) > 0 { + if documentCounts, err = s.idx.DocumentCountsByAuthorSlugs(slugs); err != nil { + log.Println(err) + return fiber.ErrInternalServerError + } + } + + keywords := searchFields.Name + templateVars := s.baseTemplateVars(c, TypeAuthors) + templateVars["SearchFields"] = searchFields + templateVars["AuthorSearchFields"] = searchFields + templateVars["DocumentSearchFields"] = index.SearchFields{Keywords: keywords} + templateVars["SearchQuery"] = keywords + templateVars["SelectedGender"] = c.Query("gender") + templateVars["Results"] = authorResults + templateVars["DocumentCounts"] = documentCounts + templateVars["Title"] = "Search authors" + templateVars["SortBy"] = c.Query("sort-by") + templateVars["AdditionalSortOptions"] = authorSortOptions() + + return s.renderSearch(c, templateVars, "partials/authors-list-fragments") +} + +func (s *Controller) baseTemplateVars(c fiber.Ctx, searchType string) fiber.Map { + return fiber.Map{ + "SearchType": searchType, + "SearchPage": true, + "EmailFrom": s.sender.From(), + "URL": view.URL(c), + "SortURL": view.BaseURLWithout(c, "sort-by", "page"), + } +} + +func (s *Controller) renderSearch(c fiber.Ctx, templateVars fiber.Map, fragmentTemplate string) error { + if results, ok := templateVars["Results"]; ok { + switch r := results.(type) { + case result.Paginated[[]model.AugmentedDocument]: + templateVars["Paginator"] = view.Pagination(model.MaxPagesNavigator, r, c.Queries()) + case result.Paginated[[]index.Author]: + templateVars["Paginator"] = view.Pagination(model.MaxPagesNavigator, r, c.Queries()) + } + } + + if c.Get("hx-request") == "true" { + if err := c.Render(fragmentTemplate, templateVars); err != nil { + log.Println(err) + return fiber.ErrInternalServerError + } + return nil + } + + if err := c.Render("search/list", templateVars, "layout"); err != nil { + log.Println(err) + return fiber.ErrInternalServerError + } + + return nil +} + +func documentSortOptions() []struct { + Key string + Value string +} { + return []struct { + Key string + Value string + }{ + {"relevance", "relevance"}, + {"pub-date-older-first", "older"}, + {"pub-date-newer-first", "newer"}, + {"est-read-time-shorter-first", "shorter"}, + {"est-read-time-longer-first", "longer"}, + } +} + +func authorSortOptions() []struct { + Key string + Value string +} { + return []struct { + Key string + Value string + }{ + {"name-a-z", "name A-Z"}, + {"name-z-a", "name Z-A"}, + {"birth-older-first", "birth older first"}, + {"birth-newer-first", "birth newer first"}, + {"death-older-first", "death older first"}, + {"death-newer-first", "death newer first"}, + {"documents-more-first", "documents more first"}, + {"documents-fewer-first", "documents fewer first"}, + } +} diff --git a/internal/webserver/embedded/css/display.css b/internal/webserver/embedded/css/display.css index 0ece2ae5..0c64022a 100644 --- a/internal/webserver/embedded/css/display.css +++ b/internal/webserver/embedded/css/display.css @@ -150,6 +150,15 @@ main .form-control:focus { white-space: nowrap; } +.home-search-tab-content { + border-top-left-radius: 0; + border-top-right-radius: 0; +} + +.search-filters-panel .row { + margin-inline: 0; +} + .search-filters-sidebar-sticky { position: sticky; top: 5rem; /* clear fixed-top navbar */ @@ -282,12 +291,16 @@ main .form-control:focus { overflow: hidden; } -#searchbox-container .btn:hover { - color: var(--bs-btn-color); - background-color: rgba(0, 0, 0, 0); +#searchbox-container .input-group .form-control:focus { + box-shadow: none !important; } -#searchbox-container .input-group:focus-within { +.home-search-tabs { + --bs-nav-tabs-border-radius: var(--bs-border-radius-xl); + width: 100%; +} + +.home-search-input-group:focus-within { color: #212529; border-color: #86b7fe; outline: 0; @@ -296,12 +309,18 @@ main .form-control:focus { border-radius: .25rem; } -#searchbox-container .input-group .form-control:focus { - box-shadow: none !important; +#searchbox-container .btn:hover { + color: var(--bs-btn-color); + background-color: rgba(0, 0, 0, 0); } -#latest-docs .carousel-indicators { - position: relative; +#searchbox-container .input-group:focus-within { + color: #212529; + border-color: #86b7fe; + outline: 0; + box-shadow: 0 0 0 0.25rem rgb(13 110 253 / 25%); + z-index: 3; + border-radius: .25rem; } /* Completion checkbox and date styles */ diff --git a/internal/webserver/embedded/js/author-search-filters.js b/internal/webserver/embedded/js/author-search-filters.js new file mode 100644 index 00000000..4636d52a --- /dev/null +++ b/internal/webserver/embedded/js/author-search-filters.js @@ -0,0 +1,50 @@ +"use strict" + +import { + applyHiddenDatesToVisible, + bindOffcanvasFilterSync, + enableFilterInputsOnPageShow, + initDateControls, + initFilterFormBehavior, + syncSidebarFormToOffcanvas, +} from './search-filter-utils.js' + +function authorSyncOffcanvas() { + syncSidebarFormToOffcanvas({ + searchFieldName: 'name', + offcanvasContainerId: 'author-search-filters', + }) +} + +function initAuthorSearchFilters(searchFilters) { + if (!searchFilters) return + const searchFiltersForm = searchFilters.closest('form') + if (!searchFiltersForm) return + + const composeDateControls = initDateControls(searchFilters, searchFiltersForm) + searchFiltersForm._coreanderComposeDates = searchFiltersForm._coreanderComposeDates || [] + searchFiltersForm._coreanderComposeDates.push(composeDateControls) + if (searchFiltersForm.dataset.coreanderFilterBehavior === 'true') { + return + } + initFilterFormBehavior({ + searchFilters, + searchFiltersForm, + composeDateControls, + listPath: '/search', + syncOffcanvas: authorSyncOffcanvas, + }) + searchFiltersForm.dataset.coreanderFilterBehavior = 'true' +} + +enableFilterInputsOnPageShow(['author-search-filters', 'author-search-filters-sidebar']) + +initAuthorSearchFilters(document.getElementById('author-search-filters')) +initAuthorSearchFilters(document.getElementById('author-search-filters-sidebar')) + +bindOffcanvasFilterSync({ + sidebarFormId: 'search-filters-form', + offcanvasContainerId: 'author-search-filters', + offcanvasElementId: 'search-filters-offcanvas', + syncOffcanvas: authorSyncOffcanvas, +}) diff --git a/internal/webserver/embedded/js/document-search-filters.js b/internal/webserver/embedded/js/document-search-filters.js new file mode 100644 index 00000000..80183f93 --- /dev/null +++ b/internal/webserver/embedded/js/document-search-filters.js @@ -0,0 +1,247 @@ +"use strict" + +import { + bindOffcanvasFilterSync, + enableFilterInputsOnPageShow, + initDateControls, + initFilterFormBehavior, + syncSidebarFormToOffcanvas, +} from './search-filter-utils.js' + +// Load translations (subjects UI) +let translations = {} +const i18nElement = document.getElementById('i18n') +if (i18nElement) { + try { + translations = JSON.parse(i18nElement.textContent).i18n + } catch (_) { + translations = {} + } +} + +function documentsSyncOffcanvas() { + syncSidebarFormToOffcanvas({ + searchFieldName: 'search', + offcanvasContainerId: 'document-search-filters', + afterCopy: (offcanvasContainer) => { + const sidebarSubjectsHidden = document.getElementById('sidebar-subjects-hidden') + const offcanvasSubjectsHidden = document.getElementById('subjects-hidden') + if (sidebarSubjectsHidden && offcanvasSubjectsHidden) { + offcanvasSubjectsHidden.value = sidebarSubjectsHidden.value + } + offcanvasContainer.dispatchEvent(new CustomEvent('syncSubjectsFromHiddenInput')) + }, + }) +} + +function initDocumentSearchFilters(searchFilters) { + if (!searchFilters) return + const searchFiltersForm = searchFilters.closest('form') + if (!searchFiltersForm) return + + const idPrefix = searchFilters.id === 'document-search-filters-sidebar' ? 'sidebar-' : '' + const composeDateControls = initDateControls(searchFilters, searchFiltersForm) + + if (searchFiltersForm.dataset.coreanderFilterBehavior !== 'true') { + const { scheduleApplyFilters } = initFilterFormBehavior({ + searchFilters, + searchFiltersForm, + composeDateControls, + listPath: '/search', + syncOffcanvas: documentsSyncOffcanvas, + beforeSidebarApply: () => { + const sidebarContainer = document.getElementById('document-search-filters-sidebar') + if (sidebarContainer) sidebarContainer.dispatchEvent(new CustomEvent('syncSubjectsFromHiddenInput')) + }, + }) + searchFiltersForm.dataset.coreanderFilterBehavior = 'true' + initSubjectsFilters(searchFilters, idPrefix, scheduleApplyFilters) + return + } + + initSubjectsFilters(searchFilters, idPrefix, null) +} + +export function initSubjectsFilters(searchFilters, idPrefix, triggerSearchUpdate) { + const subjectsList = document.getElementById(idPrefix + 'subjects-list') + const subjectsInput = document.getElementById(idPrefix + 'subjects') + const subjectsHiddenInput = document.getElementById(idPrefix + 'subjects-hidden') + const subjectsBadgesContainer = document.getElementById(idPrefix + 'subjects-badges-container') + let selectedSubjectSlugs = [] + let slugToNames = {} + let nameToSlug = {} + + if (subjectsList) { + fetch('/subjects') + .then(response => { + if (!response.ok) { + throw new Error('Failed to fetch subjects') + } + return response.json() + }) + .then(bySlug => { + slugToNames = {} + nameToSlug = {} + subjectsList.innerHTML = '' + Object.entries(bySlug || {}).forEach(([slug, names]) => { + const nameList = names || [] + slugToNames[slug] = nameList + nameList.forEach(name => { + nameToSlug[name] = slug + }) + const displayText = nameList.join(', ') + nameToSlug[displayText] = slug + const option = document.createElement('option') + option.value = displayText + subjectsList.appendChild(option) + }) + applyInitialSubjects() + updateSubjectBadges() + }) + .catch(error => { + console.error('Error loading subjects:', error) + }) + } + + function slugForValue(value) { + const trimmed = (value || '').trim() + if (!trimmed) return null + if (nameToSlug[trimmed]) return nameToSlug[trimmed] + if (slugToNames[trimmed]) return trimmed + return null + } + + function displayNamesForSlug(slug) { + return (slugToNames[slug] || [slug]).join(', ') + } + + function updateSubjectBadges() { + if (!subjectsBadgesContainer || !subjectsHiddenInput) return + subjectsBadgesContainer.innerHTML = '' + if (selectedSubjectSlugs.length === 0) { + subjectsBadgesContainer.classList.add('d-none') + subjectsHiddenInput.value = '' + return + } + subjectsBadgesContainer.classList.remove('d-none') + selectedSubjectSlugs.forEach((slug, index) => { + const displayText = displayNamesForSlug(slug) + const badge = document.createElement('span') + badge.className = 'badge rounded-pill text-bg-primary d-inline-flex align-items-center' + badge.style.pointerEvents = 'all' + badge.textContent = displayText + const closeBtn = document.createElement('button') + closeBtn.type = 'button' + closeBtn.className = 'btn-close btn-close-white ms-1 mt-0 small' + const removeSubjectLabel = translations.remove_subject ? translations.remove_subject.replace('%s', displayText) : `Remove subject: ${displayText}` + closeBtn.setAttribute('aria-label', removeSubjectLabel) + closeBtn.addEventListener('click', (e) => { + e.preventDefault() + e.stopPropagation() + removeSubject(index) + }) + badge.appendChild(closeBtn) + subjectsBadgesContainer.appendChild(badge) + }) + subjectsHiddenInput.value = selectedSubjectSlugs.join(',') + } + + function addSubject(value) { + const slug = slugForValue(value) + if (!slug) return + const isDuplicate = selectedSubjectSlugs.includes(slug) + if (!isDuplicate) { + selectedSubjectSlugs.push(slug) + updateSubjectBadges() + if (triggerSearchUpdate) triggerSearchUpdate() + } + if (subjectsInput) subjectsInput.value = '' + } + + function removeSubject(index) { + selectedSubjectSlugs.splice(index, 1) + updateSubjectBadges() + if (triggerSearchUpdate) triggerSearchUpdate() + if (subjectsInput) subjectsInput.focus() + } + + function matchesDatalistOption(value) { + if (!subjectsList) return false + const options = Array.from(subjectsList.options) + return options.some(option => option.value === value) + } + + function handlePotentialDatalistMatch(value) { + if (!value) return + if (matchesDatalistOption(value)) { + addSubject(value) + } + } + + function applyInitialSubjects() { + if (!subjectsHiddenInput) return + const raw = subjectsHiddenInput.value + const parts = raw ? raw.split(',').map(s => s.trim()).filter(Boolean) : [] + const seen = new Set() + selectedSubjectSlugs = [] + parts.forEach(part => { + const slug = slugForValue(part) || part + const key = slug.toLowerCase() + if (!seen.has(key)) { + seen.add(key) + selectedSubjectSlugs.push(slug) + } + }) + } + + if (subjectsInput && subjectsHiddenInput) { + if (document.readyState === 'loading') { + document.addEventListener('DOMContentLoaded', () => { + if (Object.keys(slugToNames).length > 0) updateSubjectBadges() + }) + } + searchFilters.addEventListener('syncSubjectsFromHiddenInput', () => { + if (!subjectsHiddenInput) return + applyInitialSubjects() + updateSubjectBadges() + }) + let lastInputValue = '' + subjectsInput.addEventListener('input', (e) => { + const value = e.target.value.trim() + lastInputValue = value + handlePotentialDatalistMatch(value) + }) + subjectsInput.addEventListener('change', (e) => { + const value = e.target.value.trim() + if (value && value !== lastInputValue) { + addSubject(value) + } + }) + subjectsInput.addEventListener('blur', (e) => { + handlePotentialDatalistMatch(e.target.value.trim()) + }) + subjectsInput.addEventListener('keydown', (e) => { + if (e.key === 'Enter') { + e.preventDefault() + const value = subjectsInput.value.trim() + if (value) addSubject(value) + } else if (e.key === 'Backspace' && subjectsInput.value === '' && selectedSubjectSlugs.length > 0) { + removeSubject(selectedSubjectSlugs.length - 1) + } + }) + } +} + +if (!document.getElementById('home-search-form')) { + enableFilterInputsOnPageShow(['document-search-filters', 'document-search-filters-sidebar']) + + initDocumentSearchFilters(document.getElementById('document-search-filters')) + initDocumentSearchFilters(document.getElementById('document-search-filters-sidebar')) + + bindOffcanvasFilterSync({ + sidebarFormId: 'search-filters-form', + offcanvasContainerId: 'document-search-filters', + offcanvasElementId: 'search-filters-offcanvas', + syncOffcanvas: documentsSyncOffcanvas, + }) +} diff --git a/internal/webserver/embedded/js/home-search.js b/internal/webserver/embedded/js/home-search.js new file mode 100644 index 00000000..0fdba888 --- /dev/null +++ b/internal/webserver/embedded/js/home-search.js @@ -0,0 +1,178 @@ +"use strict" + +import { initDateControls, syncSearchTypeFromPane } from './search-filter-utils.js' + +const STORAGE_KEY = "coreander-home-search-tab" + +function getStoredTab() { + try { + const stored = sessionStorage.getItem(STORAGE_KEY) + if (stored === "documents" || stored === "authors") { + return stored + } + } catch (_) { + // ignore storage errors + } + return "documents" +} + +function setHomeSearchType(type) { + const typeInput = document.getElementById("home-search-type") + if (typeInput) typeInput.value = type +} + +function setHomeActivePanelInputs(activeType) { + const docPanel = document.getElementById("home-search-documents-panel") + const authorPanel = document.getElementById("home-search-authors-panel") + + docPanel?.querySelectorAll("input, select, textarea").forEach((el) => { + if (el.id === "home-search-type") return + el.disabled = activeType !== "documents" + }) + authorPanel?.querySelectorAll("input, select, textarea").forEach((el) => { + el.disabled = activeType !== "authors" + }) +} + +function isAdvancedSearchOpen(collapse) { + if (collapse?.classList.contains("show")) return true + const toggle = document.querySelector('[href="#home-advanced-search-collapse"], [data-bs-target="#home-advanced-search-collapse"]') + return toggle?.getAttribute("aria-expanded") === "true" +} + +function resolveSearchType(collapse) { + if (isAdvancedSearchOpen(collapse)) { + return syncSearchTypeFromPane("home-search-type", "home-search-authors-panel") + } + setHomeSearchType("documents") + return "documents" +} + +function hasFilterParams(params) { + for (const key of params.keys()) { + if (key !== "type") return true + } + return false +} + +function collectPanelParams(panel, composeDateControls, type) { + if (!panel) return new URLSearchParams() + + composeDateControls() + const params = new URLSearchParams() + params.set("type", type) + panel.querySelectorAll("input, select, textarea").forEach((el) => { + if (!el.name || el.disabled) return + if ((el.type === "checkbox" || el.type === "radio") && !el.checked) return + const value = String(el.value ?? "").trim() + if (value === "" || value === "0") return + params.append(el.name, value) + }) + return params +} + +function tabTypeFromEvent(event) { + const tabBtn = event.target?.closest?.("[data-home-search-tab]") ?? event.target + const tabName = tabBtn?.dataset?.homeSearchTab + return tabName === "authors" || tabName === "documents" ? tabName : null +} + +function restoreStoredTab() { + try { + const stored = sessionStorage.getItem(STORAGE_KEY) + if (stored !== "authors") return + const authorsTab = document.getElementById("home-search-tab-authors") + if (authorsTab && typeof bootstrap !== "undefined") { + bootstrap.Tab.getOrCreateInstance(authorsTab).show() + } + } catch (_) { + // ignore storage errors + } +} + +function initHomeSearchTabs() { + const tabs = document.getElementById("home-search-tabs") + if (!tabs) return + + let activeType = getStoredTab() + setHomeSearchType(activeType) + setHomeActivePanelInputs(activeType) + + tabs.addEventListener("show.bs.tab", (event) => { + const tabName = tabTypeFromEvent(event) + if (!tabName) return + activeType = tabName + setHomeSearchType(tabName) + setHomeActivePanelInputs(tabName) + }) + + tabs.addEventListener("shown.bs.tab", (event) => { + const tabName = tabTypeFromEvent(event) + if (!tabName) return + try { + sessionStorage.setItem(STORAGE_KEY, tabName) + } catch (_) { + // ignore storage errors + } + }) + + restoreStoredTab() + activeType = syncSearchTypeFromPane("home-search-type", "home-search-authors-panel") + setHomeActivePanelInputs(activeType) +} + +function safeInitDateControls(panel, form) { + if (!panel) return () => {} + try { + return initDateControls(panel, form) + } catch (error) { + console.error("Error initializing date controls:", error) + return () => {} + } +} + +function initHomeSearch() { + const form = document.getElementById("home-search-form") + if (!form) return + + const collapse = document.getElementById("home-advanced-search-collapse") + const docPanel = document.getElementById("home-search-documents-panel") + const authorPanel = document.getElementById("home-search-authors-panel") + const searchbox = form.querySelector("#searchbox") + + const composeDocumentDates = safeInitDateControls(docPanel, form) + const composeAuthorDates = safeInitDateControls(authorPanel, form) + form._coreanderComposeDates = [composeDocumentDates, composeAuthorDates] + + form.addEventListener("submit", (event) => { + event.preventDefault() + const query = searchbox?.value.trim() ?? "" + const searchType = resolveSearchType(collapse) + + if (!isAdvancedSearchOpen(collapse)) { + if (!query) return + const params = new URLSearchParams({ type: searchType, search: query }) + window.location.href = "/search?" + params.toString() + return + } + + const composeDates = searchType === "authors" ? composeAuthorDates : composeDocumentDates + const panel = searchType === "authors" ? authorPanel : docPanel + + const params = collectPanelParams(panel, composeDates, searchType) + if (query) params.set("search", query) + if (!query && !params.has("name") && !hasFilterParams(params)) return + window.location.href = "/search?" + params.toString() + }) + + initHomeSearchTabs() + + const docFilters = docPanel?.querySelector("#document-search-filters") + if (docFilters) { + import("./document-search-filters.js") + .then(({ initSubjectsFilters }) => initSubjectsFilters(docFilters, "", null)) + .catch(() => {}) + } +} + +initHomeSearch() diff --git a/internal/webserver/embedded/js/search-filter-utils.js b/internal/webserver/embedded/js/search-filter-utils.js new file mode 100644 index 00000000..14869646 --- /dev/null +++ b/internal/webserver/embedded/js/search-filter-utils.js @@ -0,0 +1,300 @@ +"use strict" + +function isLeapYear(year) { + return (year % 4 === 0) && (year % 100 !== 0 || year % 400 === 0) +} + +export function updateHiddenDateInput(dateControl) { + const yearInput = dateControl.querySelector('.input-year') + const monthSelect = dateControl.querySelector('.input-month') + const dayInput = dateControl.querySelector('.input-day') + const hiddenDateInput = dateControl.parentElement.querySelector('.date') + + if (!yearInput.value || yearInput.value === '' || yearInput.value === '0') { + hiddenDateInput.value = '' + return + } + + let year = yearInput.value + if (year.startsWith('-') || year.startsWith('+')) { + year = year.substring(0, 1) + year.substring(1).padStart(4, '0') + } else { + year = year.padStart(4, '0') + } + + const month = monthSelect.value || '01' + const day = (dayInput.value || '1').padStart(2, '0') + + hiddenDateInput.value = year + '-' + month + '-' + day +} + +function updateMaxDays(monthSelect, dayInput, yearInput, dateControl = null) { + const month = parseInt(monthSelect.value) + const year = parseInt(yearInput.value) || new Date().getFullYear() + + let maxDays = 31 + switch (month) { + case 2: + maxDays = isLeapYear(year) ? 29 : 28 + break + case 4: + case 6: + case 9: + case 11: + maxDays = 30 + break + } + + dayInput.setAttribute('max', maxDays) + + const currentDay = parseInt(dayInput.value) + if (currentDay > maxDays) { + dayInput.value = maxDays + if (dateControl) { + updateHiddenDateInput(dateControl) + } + } +} + +export function copyFormValues(sourceForm, targetForm) { + for (const el of sourceForm.elements) { + if (!el.name) continue + const target = targetForm.elements[el.name] + if (target && target !== el) { + if (target.type === 'checkbox' || target.type === 'radio') { + target.checked = el.checked + } else { + target.value = el.value + } + } + } +} + +function yearForDisplay(yearStr) { + if (!yearStr) return '' + if (yearStr.startsWith('-')) { + const rest = yearStr.slice(1).replace(/^0+/, '') + return rest === '' ? '' : '-' + rest + } + const stripped = yearStr.replace(/^0+/, '') + return stripped === '' ? '' : stripped +} + +export function applyHiddenDatesToVisible(container) { + if (!container) return + container.querySelectorAll('.date-control').forEach(dateControl => { + const hiddenInput = dateControl.parentElement.querySelector('input.date') + if (!hiddenInput || !hiddenInput.value) return + const parts = hiddenInput.value.split('-') + if (parts.length < 3) return + const yearInput = dateControl.querySelector('.input-year') + const monthSelect = dateControl.querySelector('.input-month') + const dayInput = dateControl.querySelector('.input-day') + if (yearInput) yearInput.value = yearForDisplay(parts[0]) + if (monthSelect) monthSelect.value = parts[1] + if (dayInput) dayInput.value = String(parseInt(parts[2], 10)) + }) +} + +export function initDateControls(searchFilters, searchFiltersForm) { + searchFilters.querySelectorAll('.date-control').forEach(dateControl => { + const monthSelect = dateControl.querySelector('.input-month') + const dayInput = dateControl.querySelector('.input-day') + const yearInput = dateControl.querySelector('.input-year') + if (!monthSelect || !dayInput || !yearInput) return + + monthSelect.addEventListener('change', () => { + updateMaxDays(monthSelect, dayInput, yearInput, dateControl) + updateHiddenDateInput(dateControl) + }) + + yearInput.addEventListener('change', () => { + if (parseInt(monthSelect.value) === 2) { + updateMaxDays(monthSelect, dayInput, yearInput, dateControl) + } + updateHiddenDateInput(dateControl) + }) + + yearInput.addEventListener('input', () => { + updateHiddenDateInput(dateControl) + }) + + dayInput.addEventListener('change', () => { + updateHiddenDateInput(dateControl) + }) + + dayInput.addEventListener('input', () => { + updateHiddenDateInput(dateControl) + }) + + updateMaxDays(monthSelect, dayInput, yearInput, dateControl) + updateHiddenDateInput(dateControl) + }) + + return function composeDateControls() { + searchFiltersForm.querySelectorAll('.date-control').forEach(el => { + const yearEl = el.querySelector('.input-year') + if (!yearEl || yearEl.value === '' || yearEl.value === '0') return + const composed = el.parentElement.querySelector('.date') + if (!composed) return + let year = yearEl.value + if (year.startsWith('-') || year.startsWith('+')) { + year = year.substring(0, 1) + year.substring(1).padStart(4, '0') + } else { + year = year.padStart(4, '0') + } + const month = el.querySelector('.input-month').value || '01' + const day = (el.querySelector('.input-day').value || '1').padStart(2, '0') + composed.value = year + '-' + month + '-' + day + }) + } +} + +export function syncSidebarFormToOffcanvas({ searchFieldName, offcanvasContainerId, afterCopy }) { + const sidebarForm = document.getElementById('search-filters-form') + const offcanvasContainer = document.getElementById(offcanvasContainerId) + if (!sidebarForm) return + + const field = sidebarForm.elements[searchFieldName] + const searchValue = field ? field.value : '' + const navSearchbox = document.getElementById('searchbox') + if (navSearchbox) navSearchbox.value = searchValue + + if (!offcanvasContainer) return + const offcanvasForm = offcanvasContainer.closest('form') + if (!offcanvasForm) return + + copyFormValues(sidebarForm, offcanvasForm) + if (afterCopy) afterCopy(offcanvasContainer, sidebarForm) + applyHiddenDatesToVisible(offcanvasContainer) +} + +const FILTER_DEBOUNCE_MS = 600 + +const SEARCH_LIST_PATHS = new Set(['/search', '/documents', '/authors']) + +export function syncSearchTypeFromPane(typeInputId, authorPaneId) { + const typeInput = document.getElementById(typeInputId) + const authorPane = document.getElementById(authorPaneId) + const type = authorPane?.classList.contains('active') ? 'authors' : 'documents' + if (typeInput) typeInput.value = type + return type +} + +export function syncSidebarSearchTypeFromPane() { + return syncSearchTypeFromPane('search-type', 'search-sidebar-authors-panel') +} + +function composeAllDateControls(form, fallbackCompose) { + const composers = form?._coreanderComposeDates + if (composers?.length) { + composers.forEach((fn) => fn()) + return + } + fallbackCompose() +} + +function activeSearchListPath(fallbackPath) { + if (document.getElementById('search-filters-form')) { + return '/search' + } + if (SEARCH_LIST_PATHS.has(window.location.pathname)) { + return window.location.pathname + } + return fallbackPath +} + +export function initFilterFormBehavior({ + searchFilters, + searchFiltersForm, + composeDateControls, + listPath, + syncOffcanvas, + beforeSidebarApply, +}) { + const resolvedListPath = activeSearchListPath(listPath) + const isListPage = SEARCH_LIST_PATHS.has(window.location.pathname) && document.getElementById('search-filters-form') + let applyingFilters = false + + function applyFilters() { + applyingFilters = true + const sidebarForm = document.getElementById('search-filters-form') + if (sidebarForm && isListPage) { + syncSidebarSearchTypeFromPane() + composeAllDateControls(sidebarForm, composeDateControls) + if (searchFiltersForm !== sidebarForm) { + copyFormValues(searchFiltersForm, sidebarForm) + if (beforeSidebarApply) beforeSidebarApply() + } + const formData = new FormData(sidebarForm) + const params = new URLSearchParams() + for (const [k, v] of formData.entries()) { + if (v != null && String(v).trim() !== '') params.append(k, v) + } + const queryString = params.toString() + const url = resolvedListPath + (queryString ? '?' + queryString : '') + history.replaceState(null, '', url) + window.htmx.trigger(document.body, 'update') + syncOffcanvas() + } else { + composeAllDateControls(searchFiltersForm, composeDateControls) + const params = new URLSearchParams(new FormData(searchFiltersForm)) + window.location.href = resolvedListPath + '?' + params.toString() + } + setTimeout(() => { applyingFilters = false }, 0) + } + + let scheduleApplyFilters = null + if (isListPage) { + let applyFiltersDebounced + scheduleApplyFilters = function () { + if (applyingFilters) return + if (applyFiltersDebounced) clearTimeout(applyFiltersDebounced) + applyFiltersDebounced = setTimeout(applyFilters, FILTER_DEBOUNCE_MS) + } + + searchFiltersForm.addEventListener('submit', (e) => { + e.preventDefault() + if (applyFiltersDebounced) clearTimeout(applyFiltersDebounced) + applyFilters() + }) + + searchFiltersForm.addEventListener('input', () => scheduleApplyFilters()) + searchFiltersForm.addEventListener('change', () => scheduleApplyFilters()) + } else { + searchFiltersForm.addEventListener('submit', () => { + composeDateControls() + searchFilters.querySelectorAll('input').forEach(input => { + if (input.value === '' || input.value === '0') input.setAttribute('disabled', 'disabled') + }) + }) + } + + searchFiltersForm._coreanderComposeDates = searchFiltersForm._coreanderComposeDates || [] + searchFiltersForm._coreanderComposeDates.push(composeDateControls) + + return { scheduleApplyFilters } +} + +export function enableFilterInputsOnPageShow(containerIds) { + window.addEventListener('pageshow', () => { + containerIds.forEach(id => { + const el = document.getElementById(id) + if (el) { + el.querySelectorAll('input').forEach(input => { + input.removeAttribute('disabled') + }) + } + }) + }) +} + +export function bindOffcanvasFilterSync({ sidebarFormId, offcanvasContainerId, offcanvasElementId, syncOffcanvas }) { + if (!document.getElementById(sidebarFormId) || !document.getElementById(offcanvasContainerId)) { + return + } + const offcanvasEl = document.getElementById(offcanvasElementId) + if (offcanvasEl) { + offcanvasEl.addEventListener('shown.bs.offcanvas', () => syncOffcanvas()) + } +} diff --git a/internal/webserver/embedded/js/search-filters.js b/internal/webserver/embedded/js/search-filters.js deleted file mode 100644 index 77ba152e..00000000 --- a/internal/webserver/embedded/js/search-filters.js +++ /dev/null @@ -1,479 +0,0 @@ -"use strict" - -// Load translations (shared) -let translations = {} -const i18nElement = document.getElementById('i18n') -if (i18nElement) { - translations = JSON.parse(i18nElement.textContent).i18n -} - -/** - * Determines if a given year is a leap year - * @param {number} year - The year to check - * @returns {boolean} - True if the year is a leap year, false otherwise - */ -function isLeapYear(year) { - // A year is a leap year if: - // 1. It's divisible by 4 AND - // 2. It's either NOT divisible by 100 OR it's divisible by 400 - return (year % 4 === 0) && (year % 100 !== 0 || year % 400 === 0) -} - -/** - * Updates the max attribute of the day input based on the selected month and year - * @param {HTMLElement} monthSelect - The month select element - * @param {HTMLElement} dayInput - The day input element - * @param {HTMLElement} yearInput - The year input element - * @param {HTMLElement} dateControl - The date-control container element (optional, for updating hidden input) - */ -function updateMaxDays(monthSelect, dayInput, yearInput, dateControl = null) { - const month = parseInt(monthSelect.value) - const year = parseInt(yearInput.value) || new Date().getFullYear() - - let maxDays = 31 // default max days - - switch (month) { - case 2: // February - maxDays = isLeapYear(year) ? 29 : 28 - break - case 4: // April - case 6: // June - case 9: // September - case 11: // November - maxDays = 30 - break - } - - // Update the max attribute - dayInput.setAttribute('max', maxDays) - - // If current day value is greater than max days, set it to max days - const currentDay = parseInt(dayInput.value) - if (currentDay > maxDays) { - dayInput.value = maxDays - // Update hidden input if dateControl is provided - if (dateControl) { - updateHiddenDateInput(dateControl) - } - } -} - -/** - * Updates the hidden date input field with the composed date value - * @param {HTMLElement} dateControl - The date-control container element - */ -function updateHiddenDateInput(dateControl) { - const yearInput = dateControl.querySelector('.input-year') - const monthSelect = dateControl.querySelector('.input-month') - const dayInput = dateControl.querySelector('.input-day') - const hiddenDateInput = dateControl.parentElement.querySelector('.date') - - // Only update if year has a value - if (!yearInput.value || yearInput.value === '' || yearInput.value === '0') { - hiddenDateInput.value = '' - return - } - - let year = yearInput.value - if (year.startsWith('-') || year.startsWith('+')) { - year = year.substring(0, 1) + year.substring(1).padStart(4, '0') - } else { - year = year.padStart(4, '0') - } - - const month = monthSelect.value || '01' - const day = (dayInput.value || '1').padStart(2, '0') - - hiddenDateInput.value = year + '-' + month + '-' + day -} - -/** - * Initialize search filters for a single container (main or sidebar). - * @param {HTMLElement} searchFilters - The container element (#search-filters or #search-filters-sidebar) - */ -function initSearchFilters(searchFilters) { - if (!searchFilters) return - const searchFiltersForm = searchFilters.closest('form') - if (!searchFiltersForm) return - - const idPrefix = searchFilters.id === 'search-filters-sidebar' ? 'sidebar-' : '' - - // Set up event listeners for all month selects - searchFilters.querySelectorAll('.date-control').forEach(dateControl => { - const monthSelect = dateControl.querySelector('.input-month') - const dayInput = dateControl.querySelector('.input-day') - const yearInput = dateControl.querySelector('.input-year') - - // Update max days when month changes - monthSelect.addEventListener('change', () => { - updateMaxDays(monthSelect, dayInput, yearInput, dateControl) - updateHiddenDateInput(dateControl) - }) - - // Update max days when year changes (for February) - yearInput.addEventListener('change', () => { - if (parseInt(monthSelect.value) === 2) { - updateMaxDays(monthSelect, dayInput, yearInput, dateControl) - } - updateHiddenDateInput(dateControl) - }) - - yearInput.addEventListener('input', () => { - updateHiddenDateInput(dateControl) - }) - - dayInput.addEventListener('change', () => { - updateHiddenDateInput(dateControl) - }) - - dayInput.addEventListener('input', () => { - updateHiddenDateInput(dateControl) - }) - - updateMaxDays(monthSelect, dayInput, yearInput, dateControl) - updateHiddenDateInput(dateControl) - }) - - function composeDateControls() { - searchFiltersForm.querySelectorAll('.date-control').forEach(function (el) { - const yearEl = el.querySelector('.input-year') - if (!yearEl || (yearEl.value === '' || yearEl.value === '0')) return - const composed = el.parentElement.querySelector('.date') - if (!composed) return - let year = yearEl.value - if (year.startsWith('-') || year.startsWith('+')) { - year = year.substring(0, 1) + year.substring(1).padStart(4, '0') - } else { - year = year.padStart(4, '0') - } - const month = el.querySelector('.input-month').value || '01' - const day = (el.querySelector('.input-day').value || '1').padStart(2, '0') - composed.value = year + '-' + month + '-' + day - }) - } - - const isDocumentsPage = window.location.pathname === '/documents' - - let applyingFilters = false - - function applyFilters() { - applyingFilters = true - composeDateControls() - const sidebarForm = document.getElementById('search-filters-form') - if (sidebarForm && isDocumentsPage) { - if (searchFiltersForm !== sidebarForm) { - copyFormValues(searchFiltersForm, sidebarForm) - const sidebarContainer = document.getElementById('search-filters-sidebar') - if (sidebarContainer) sidebarContainer.dispatchEvent(new CustomEvent('syncSubjectsFromHiddenInput')) - } - const formData = new FormData(sidebarForm) - const params = new URLSearchParams() - for (const [k, v] of formData.entries()) { - if (v != null && String(v).trim() !== '') params.append(k, v) - } - const queryString = params.toString() - const url = '/documents' + (queryString ? '?' + queryString : '') - window.htmx.trigger(document.body, 'update') - history.replaceState(null, '', url) - syncSidebarFormToOffcanvas() - } else { - const params = new URLSearchParams(new FormData(searchFiltersForm)) - window.location.href = '/documents?' + params.toString() - } - setTimeout(() => { applyingFilters = false }, 0) - } - - let triggerSearchUpdate = null - if (isDocumentsPage) { - const FILTER_DEBOUNCE_MS = 600 - let applyFiltersDebounced - function scheduleApplyFilters() { - if (applyingFilters) return - if (applyFiltersDebounced) clearTimeout(applyFiltersDebounced) - applyFiltersDebounced = setTimeout(applyFilters, FILTER_DEBOUNCE_MS) - } - triggerSearchUpdate = scheduleApplyFilters - - searchFiltersForm.addEventListener('submit', (e) => { - e.preventDefault() - if (applyFiltersDebounced) clearTimeout(applyFiltersDebounced) - applyFilters() - }) - - searchFiltersForm.addEventListener('input', () => scheduleApplyFilters()) - searchFiltersForm.addEventListener('change', () => scheduleApplyFilters()) - } else { - searchFiltersForm.addEventListener('submit', () => { - composeDateControls() - searchFilters.querySelectorAll('input').forEach(input => { - if (input.value === '' || input.value === '0') input.setAttribute('disabled', 'disabled') - }) - }) - } - - // Subjects (scoped to this container): grouped by slug; selection stores slugs, badges show all names for that slug - const subjectsList = document.getElementById(idPrefix + 'subjects-list') - const subjectsInput = document.getElementById(idPrefix + 'subjects') - const subjectsHiddenInput = document.getElementById(idPrefix + 'subjects-hidden') - const subjectsBadgesContainer = document.getElementById(idPrefix + 'subjects-badges-container') - let selectedSubjectSlugs = [] - let slugToNames = {} - let nameToSlug = {} - - if (subjectsList) { - fetch('/subjects') - .then(response => { - if (!response.ok) { - throw new Error('Failed to fetch subjects') - } - return response.json() - }) - .then(bySlug => { - slugToNames = {} - nameToSlug = {} - subjectsList.innerHTML = '' - Object.entries(bySlug || {}).forEach(([slug, names]) => { - const nameList = names || [] - slugToNames[slug] = nameList - nameList.forEach(name => { - nameToSlug[name] = slug - }) - const displayText = nameList.join(', ') - nameToSlug[displayText] = slug - const option = document.createElement('option') - option.value = displayText - subjectsList.appendChild(option) - }) - applyInitialSubjects() - updateSubjectBadges() - }) - .catch(error => { - console.error('Error loading subjects:', error) - }) - } - - function slugForValue(value) { - const trimmed = (value || '').trim() - if (!trimmed) return null - if (nameToSlug[trimmed]) return nameToSlug[trimmed] - if (slugToNames[trimmed]) return trimmed - return null - } - - function displayNamesForSlug(slug) { - return (slugToNames[slug] || [slug]).join(', ') - } - - function updateSubjectBadges() { - if (!subjectsBadgesContainer || !subjectsHiddenInput) return - subjectsBadgesContainer.innerHTML = '' - if (selectedSubjectSlugs.length === 0) { - subjectsBadgesContainer.classList.add('d-none') - subjectsHiddenInput.value = '' - return - } - subjectsBadgesContainer.classList.remove('d-none') - selectedSubjectSlugs.forEach((slug, index) => { - const displayText = displayNamesForSlug(slug) - const badge = document.createElement('span') - badge.className = 'badge rounded-pill text-bg-primary d-inline-flex align-items-center' - badge.style.pointerEvents = 'all' - badge.textContent = displayText - const closeBtn = document.createElement('button') - closeBtn.type = 'button' - closeBtn.className = 'btn-close btn-close-white ms-1 mt-0 small' - const removeSubjectLabel = translations.remove_subject ? translations.remove_subject.replace('%s', displayText) : `Remove subject: ${displayText}` - closeBtn.setAttribute('aria-label', removeSubjectLabel) - closeBtn.addEventListener('click', (e) => { - e.preventDefault() - e.stopPropagation() - removeSubject(index) - }) - badge.appendChild(closeBtn) - subjectsBadgesContainer.appendChild(badge) - }) - subjectsHiddenInput.value = selectedSubjectSlugs.join(',') - } - - function addSubject(value) { - const slug = slugForValue(value) - if (!slug) return - const isDuplicate = selectedSubjectSlugs.includes(slug) - if (!isDuplicate) { - selectedSubjectSlugs.push(slug) - updateSubjectBadges() - if (triggerSearchUpdate) triggerSearchUpdate() - } - if (subjectsInput) subjectsInput.value = '' - } - - function removeSubject(index) { - selectedSubjectSlugs.splice(index, 1) - updateSubjectBadges() - if (triggerSearchUpdate) triggerSearchUpdate() - if (subjectsInput) subjectsInput.focus() - } - - function matchesDatalistOption(value) { - if (!subjectsList) return false - const options = Array.from(subjectsList.options) - return options.some(option => option.value === value) - } - - function handlePotentialDatalistMatch(value) { - if (!value) return - if (matchesDatalistOption(value)) { - addSubject(value) - } - } - - function applyInitialSubjects() { - if (!subjectsHiddenInput) return - const raw = subjectsHiddenInput.value - const parts = raw ? raw.split(',').map(s => s.trim()).filter(Boolean) : [] - const seen = new Set() - selectedSubjectSlugs = [] - parts.forEach(part => { - const slug = slugForValue(part) || part - const key = slug.toLowerCase() - if (!seen.has(key)) { - seen.add(key) - selectedSubjectSlugs.push(slug) - } - }) - } - - if (subjectsInput && subjectsHiddenInput) { - if (document.readyState === 'loading') { - document.addEventListener('DOMContentLoaded', () => { - if (Object.keys(slugToNames).length > 0) updateSubjectBadges() - }) - } - searchFilters.addEventListener('syncSubjectsFromHiddenInput', () => { - if (!subjectsHiddenInput) return - applyInitialSubjects() - updateSubjectBadges() - }) - let lastInputValue = '' - subjectsInput.addEventListener('input', (e) => { - const value = e.target.value.trim() - lastInputValue = value - handlePotentialDatalistMatch(value) - }) - subjectsInput.addEventListener('change', (e) => { - const value = e.target.value.trim() - if (value && value !== lastInputValue) { - addSubject(value) - } - }) - subjectsInput.addEventListener('blur', (e) => { - handlePotentialDatalistMatch(e.target.value.trim()) - }) - subjectsInput.addEventListener('keydown', (e) => { - if (e.key === 'Enter') { - e.preventDefault() - const value = subjectsInput.value.trim() - if (value) addSubject(value) - } else if (e.key === 'Backspace' && subjectsInput.value === '' && selectedSubjectSlugs.length > 0) { - removeSubject(selectedSubjectSlugs.length - 1) - } - }) - } -} - -// Enable inputs when the page is shown -window.addEventListener('pageshow', () => { - ['search-filters', 'search-filters-sidebar'].forEach(id => { - const el = document.getElementById(id) - if (el) { - el.querySelectorAll('input').forEach(input => { - input.removeAttribute('disabled') - }) - } - }) -}) - -/** - * Copy form values from source to target form (by field name). - */ -function copyFormValues(sourceForm, targetForm) { - for (const el of sourceForm.elements) { - if (!el.name) continue - const target = targetForm.elements[el.name] - if (target && target !== el) { - if (target.type === 'checkbox' || target.type === 'radio') { - target.checked = el.checked - } else { - target.value = el.value - } - } - } -} - -/** - * Strip leading zeroes from a year string so "0000" / "0001" become "" / "1", and "2024" stays "2024". - * Preserves negative years (e.g. "-0500" -> "-500"). - */ -function yearForDisplay(yearStr) { - if (!yearStr) return '' - if (yearStr.startsWith('-')) { - const rest = yearStr.slice(1).replace(/^0+/, '') - return rest === '' ? '' : '-' + rest - } - const stripped = yearStr.replace(/^0+/, '') - return stripped === '' ? '' : stripped -} - -/** - * Apply hidden date input values (YYYY-MM-DD) to the visible year/month/day inputs in each .date-control. - */ -function applyHiddenDatesToVisible(container) { - if (!container) return - container.querySelectorAll('.date-control').forEach(dateControl => { - const hiddenInput = dateControl.parentElement.querySelector('input.date') - if (!hiddenInput || !hiddenInput.value) return - const parts = hiddenInput.value.split('-') - if (parts.length < 3) return - const yearInput = dateControl.querySelector('.input-year') - const monthSelect = dateControl.querySelector('.input-month') - const dayInput = dateControl.querySelector('.input-day') - if (yearInput) yearInput.value = yearForDisplay(parts[0]) - if (monthSelect) monthSelect.value = parts[1] - if (dayInput) dayInput.value = String(parseInt(parts[2], 10)) - }) -} - -/** - * Sync sidebar filter form state to the offcanvas form and navbar searchbox. - */ -function syncSidebarFormToOffcanvas() { - const sidebarForm = document.getElementById('search-filters-form') - const offcanvasContainer = document.getElementById('search-filters') - if (!sidebarForm) return - const searchValue = sidebarForm.elements['search'] ? sidebarForm.elements['search'].value : '' - const navSearchbox = document.getElementById('searchbox') - if (navSearchbox) navSearchbox.value = searchValue - if (!offcanvasContainer) return - const offcanvasForm = offcanvasContainer.closest('form') - if (!offcanvasForm) return - copyFormValues(sidebarForm, offcanvasForm) - const sidebarSubjectsHidden = document.getElementById('sidebar-subjects-hidden') - const offcanvasSubjectsHidden = document.getElementById('subjects-hidden') - if (sidebarSubjectsHidden && offcanvasSubjectsHidden) { - offcanvasSubjectsHidden.value = sidebarSubjectsHidden.value - } - applyHiddenDatesToVisible(offcanvasContainer) - offcanvasContainer.dispatchEvent(new CustomEvent('syncSubjectsFromHiddenInput')) -} - -// Initialize all filter containers on the page -initSearchFilters(document.getElementById('search-filters')) -initSearchFilters(document.getElementById('search-filters-sidebar')) - -// Keep sidebar and offcanvas filters in sync on the documents page -if (document.getElementById('search-filters-form') && document.getElementById('search-filters')) { - const offcanvasEl = document.getElementById('search-filters-offcanvas') - if (offcanvasEl) { - offcanvasEl.addEventListener('shown.bs.offcanvas', () => syncSidebarFormToOffcanvas()) - } -} diff --git a/internal/webserver/embedded/js/search-offcanvas-tabs.js b/internal/webserver/embedded/js/search-offcanvas-tabs.js new file mode 100644 index 00000000..8b9d033a --- /dev/null +++ b/internal/webserver/embedded/js/search-offcanvas-tabs.js @@ -0,0 +1,55 @@ +"use strict" + +function setOffcanvasSearchType(type) { + const typeInput = document.getElementById("search-offcanvas-type") + if (typeInput) typeInput.value = type +} + +function tabTypeFromEvent(event) { + const tabBtn = event.target?.closest?.("[data-search-tab]") ?? event.target + const tabName = tabBtn?.dataset?.searchTab + return tabName === "authors" || tabName === "documents" ? tabName : null +} + +function syncOffcanvasSearchTypeFromPane() { + const authorPane = document.getElementById("search-offcanvas-authors-panel") + const type = authorPane?.classList.contains("active") ? "authors" : "documents" + setOffcanvasSearchType(type) + return type +} + +function setOffcanvasActivePanelInputs(activeType) { + const docPanel = document.getElementById("search-offcanvas-documents-panel") + const authorPanel = document.getElementById("search-offcanvas-authors-panel") + + docPanel?.querySelectorAll("input, select, textarea").forEach((el) => { + if (el.id === "search-offcanvas-type") return + el.disabled = activeType !== "documents" + }) + authorPanel?.querySelectorAll("input, select, textarea").forEach((el) => { + el.disabled = activeType !== "authors" + }) +} + +function initSearchOffcanvasTabs() { + const tabs = document.getElementById("search-offcanvas-tabs") + if (!tabs) return + + let activeType = syncOffcanvasSearchTypeFromPane() + setOffcanvasActivePanelInputs(activeType) + + tabs.addEventListener("show.bs.tab", (event) => { + const tabName = tabTypeFromEvent(event) + if (!tabName) return + activeType = tabName + setOffcanvasSearchType(tabName) + setOffcanvasActivePanelInputs(tabName) + }) + + const form = tabs.closest("form") + form?.addEventListener("submit", () => { + syncOffcanvasSearchTypeFromPane() + }) +} + +initSearchOffcanvasTabs() diff --git a/internal/webserver/embedded/js/search-sidebar-tabs.js b/internal/webserver/embedded/js/search-sidebar-tabs.js new file mode 100644 index 00000000..b92fc800 --- /dev/null +++ b/internal/webserver/embedded/js/search-sidebar-tabs.js @@ -0,0 +1,70 @@ +"use strict" + +import { syncSidebarSearchTypeFromPane } from './search-filter-utils.js' + +function setSearchType(type) { + const typeInput = document.getElementById("search-type") + if (typeInput) typeInput.value = type +} + +function syncQueryFields(fromType, toType) { + const searchInput = document.getElementById("sidebar-search") + const nameInput = document.getElementById("sidebar-name") + if (!searchInput || !nameInput) return + if (fromType === "documents" && toType === "authors" && !nameInput.value.trim()) { + nameInput.value = searchInput.value + } + if (fromType === "authors" && toType === "documents" && !searchInput.value.trim()) { + searchInput.value = nameInput.value + } +} + +function tabTypeFromEvent(event) { + const tabBtn = event.target?.closest?.("[data-search-tab]") ?? event.target + const tabName = tabBtn?.dataset?.searchTab + return tabName === "authors" || tabName === "documents" ? tabName : null +} + +function setActivePanelInputs(activeType) { + const docPanel = document.getElementById("search-sidebar-documents-panel") + const authorPanel = document.getElementById("search-sidebar-authors-panel") + + docPanel?.querySelectorAll("input, select, textarea").forEach((el) => { + if (el.id === "search-type") return + el.disabled = activeType !== "documents" + }) + authorPanel?.querySelectorAll("input, select, textarea").forEach((el) => { + el.disabled = activeType !== "authors" + }) +} + +function initSearchSidebarTabs() { + const tabs = document.getElementById("search-sidebar-tabs") + const form = document.getElementById("search-filters-form") + if (!tabs || !form) return + + let activeType = syncSidebarSearchTypeFromPane() + setActivePanelInputs(activeType) + + tabs.addEventListener("show.bs.tab", (event) => { + const nextType = tabTypeFromEvent(event) + if (!nextType || nextType === activeType) return + syncQueryFields(activeType, nextType) + activeType = nextType + setSearchType(nextType) + setActivePanelInputs(nextType) + }) + + tabs.addEventListener("shown.bs.tab", (event) => { + const tabName = tabTypeFromEvent(event) + if (!tabName) return + activeType = tabName + setSearchType(tabName) + setActivePanelInputs(tabName) + if (window.htmx) { + window.htmx.trigger(document.body, "update") + } + }) +} + +initSearchSidebarTabs() diff --git a/internal/webserver/embedded/translations/de.yml b/internal/webserver/embedded/translations/de.yml index cd4e8dcf..6aac5a7e 100644 --- a/internal/webserver/embedded/translations/de.yml +++ b/internal/webserver/embedded/translations/de.yml @@ -1,7 +1,13 @@ "_language": "Deutsch" "Search": "Suche" +"Documents": "Dokumente" +"Authors": "Autoren" "Search results": "Suchergebnisse" "Search in %d documents": "In %d Dokumenten suchen" +"Search in %d authors": "In %d Autoren suchen" +"Search in %d documents and %d authors": "In %d Dokumenten und %d Autoren suchen" +"No results found": "Keine Ergebnisse gefunden" +"Search results": "Suchergebnisse" "Download": "Herunterladen" "%d documents found": "%d Dokumente gefunden" "No documents found": "Keine Dokumente gefunden" @@ -118,6 +124,7 @@ "Completed": "Abgeschlossen" "Completions": "Abgeschlossen" "documents": "Dokumente" +"document": "Dokument" "words": "Wörter" "All time": "Gesamte Zeit" "%s's completed documents": "Abgeschlossene Dokumente von %s" @@ -141,7 +148,12 @@ "An error occurred while uploading the image": "Beim Hochladen des Bildes ist ein Fehler aufgetreten" "Click to upload a custom image": "Klicken Sie, um ein benutzerdefiniertes Bild hochzuladen" "Indexing in progress search results may not be accurate.": "Indexierung läuft die Suchergebnisse sind möglicherweise nicht genau." +"Indexing in progress, search results may not be accurate.": "Indexierung läuft, Suchergebnisse sind möglicherweise nicht genau." +"Retrieving author information, author pages may not be complete yet.": "Autoreninformationen werden abgerufen, Autorenseiten sind möglicherweise noch unvollständig." +"Author information progress": "Fortschritt der Autoreninformationen" +"Indexing progress": "Indexierungsfortschritt" "Remaining time": "Verbleibende Zeit" +"Remaining time: %s minutes": "Verbleibende Zeit: %s Minuten" "There was an error deleting the user please try again later": "Beim Löschen des Benutzers ist ein Fehler aufgetreten bitte versuchen Sie es später erneut." "A user with this username already exists": "Ein Benutzer mit diesem Benutzernamen existiert bereits." "A user with this email address already exists": "Ein Benutzer mit dieser E-Mail-Adresse existiert bereits." @@ -165,6 +177,26 @@ "Never": "Nie" "Date of birth": "Geburtsdatum" "Date of death": "Sterbedatum" +"Search authors": "Autoren suchen" +"No authors found": "Keine Autoren gefunden" +"%d authors found": "%d Autoren gefunden" +"Name": "Name" +"Gender": "Geschlecht" +"All genders": "Alle Geschlechter" +"Male": "Männlich" +"Female": "Weiblich" +"Intersex": "Intersexuell" +"Transgender female": "Transgender weiblich" +"Transgender male": "Transgender männlich" +"Unknown": "Unbekannt" +"name A-Z": "Name A-Z" +"name Z-A": "Name Z-A" +"birth older first": "Geburt älteste zuerst" +"birth newer first": "Geburt neueste zuerst" +"death older first": "Tod älteste zuerst" +"death newer first": "Tod neueste zuerst" +"documents more first": "mehr Dokumente" +"documents fewer first": "weniger Dokumente" "%d years old": "%d Jahre alt" "BC": "v. Chr." "Before Christ": "Vor Christus" diff --git a/internal/webserver/embedded/translations/es.yml b/internal/webserver/embedded/translations/es.yml index 63e09a14..8eae34df 100644 --- a/internal/webserver/embedded/translations/es.yml +++ b/internal/webserver/embedded/translations/es.yml @@ -1,7 +1,12 @@ "_language": "Español" "Search": "Buscar" +"Documents": "Documentos" +"Authors": "Autores" "Search results": "Resultados de la búsqueda" "Search in %d documents": "Buscar en %d documentos" +"Search in %d authors": "Buscar en %d autores" +"Search in %d documents and %d authors": "Buscar en %d documentos y %d autores" +"No results found": "No se encontraron resultados" "Download": "Descargar" "%d documents found": "Hallados %d documentos" "No documents found": "No se han encontrado documentos" @@ -116,6 +121,7 @@ "Completed": "Completado" "Completions": "Completados" "documents": "documentos" +"document": "documento" "words": "palabras" "All time": "Todo el tiempo" "%s's completed documents": "Documentos completados de %s" @@ -139,6 +145,9 @@ "An error occurred while uploading the image": "Ocurrió un error al subir la imagen" "Click to upload a custom image": "Haz clic para subir una imagen personalizada" "Indexing in progress, search results may not be accurate.": "Indexando documentos, los resultados de búsqueda pueden no ser precisos." +"Retrieving author information, author pages may not be complete yet.": "Recuperando información de autores, las páginas de autor pueden estar incompletas." +"Author information progress": "Progreso de información de autores" +"Indexing progress": "Progreso de indexación" "Remaining time: %s minutes": "Tiempo restante: %s minutos" "There was an error deleting the user, please try again later": "Hubo un error al borrar el usuario, por favor, vuelva a intentarlo más tarde" "A user with this username already exists": "Ya existe un usuario con ese nombre de usuario" @@ -163,6 +172,26 @@ "Never": "Nunca" "Date of birth": "Nacimiento" "Date of death": "Fallecimiento" +"Search authors": "Buscar autores" +"No authors found": "No se encontraron autores" +"%d authors found": "%d autores encontrados" +"Name": "Nombre" +"Gender": "Género" +"All genders": "Todos los géneros" +"Male": "Masculino" +"Female": "Femenino" +"Intersex": "Intersexual" +"Transgender female": "Mujer trans" +"Transgender male": "Hombre trans" +"Unknown": "Desconocido" +"name A-Z": "nombre A-Z" +"name Z-A": "nombre Z-A" +"birth older first": "nacimiento más antiguo" +"birth newer first": "nacimiento más reciente" +"death older first": "fallecimiento más antiguo" +"death newer first": "fallecimiento más reciente" +"documents more first": "más documentos" +"documents fewer first": "menos documentos" "%d years old": "%d años" "BC": "a. C." "Before Christ": "Antes de Cristo" diff --git a/internal/webserver/embedded/translations/fr.yml b/internal/webserver/embedded/translations/fr.yml index 71c56fec..b65a8e95 100644 --- a/internal/webserver/embedded/translations/fr.yml +++ b/internal/webserver/embedded/translations/fr.yml @@ -1,7 +1,12 @@ "_language": "Français" "Search": "Rechercher" +"Documents": "Documents" +"Authors": "Auteurs" "Search results": "Résultats de la recherche" "Search in %d documents": "Rechercher dans %d documents" +"Search in %d authors": "Rechercher dans %d auteurs" +"Search in %d documents and %d authors": "Rechercher dans %d documents et %d auteurs" +"No results found": "Aucun résultat trouvé" "Download": "Télécharger" "%d documents found": "%d documents trouvés" "No documents found": "Aucun document trouvé" @@ -117,6 +122,7 @@ "Completed": "Complété" "Completions": "Complétés" "documents": "documents" +"document": "document" "words": "mots" "All time": "Tout" "%s's completed documents": "Documents complétés de %s" @@ -140,6 +146,9 @@ "An error occurred while uploading the image": "Une erreur s'est produite lors du téléchargement de l'image" "Click to upload a custom image": "Cliquez pour télécharger une image personnalisée" "Indexing in progress, search results may not be accurate.": "Indexation en cours, les résultats de la recherche peuvent ne pas être précis." +"Retrieving author information, author pages may not be complete yet.": "Récupération des informations sur les auteurs, les pages auteur peuvent être incomplètes." +"Author information progress": "Progression des informations sur les auteurs" +"Indexing progress": "Progression de l'indexation" "Remaining time: %s minutes": "Temps restant: %s minutes" "There was an error deleting the user, please try again later": "Une erreur s'est produite lors de supprimer l'utilisateur, veuillez réessayer ultérieurement" "A user with this username already exists": "Un utilisateur avec ce nom d'utilisateur existe déjà" @@ -164,6 +173,26 @@ "Never": "Jamais" "Date of birth": "Date de naissance" "Date of death": "Date de décès" +"Search authors": "Rechercher des auteurs" +"No authors found": "Aucun auteur trouvé" +"%d authors found": "%d auteurs trouvés" +"Name": "Nom" +"Gender": "Genre" +"All genders": "Tous les genres" +"Male": "Homme" +"Female": "Femme" +"Intersex": "Intersexe" +"Transgender female": "Femme transgenre" +"Transgender male": "Homme transgenre" +"Unknown": "Inconnu" +"name A-Z": "nom A-Z" +"name Z-A": "nom Z-A" +"birth older first": "naissance plus ancienne" +"birth newer first": "naissance plus récente" +"death older first": "décès plus ancien" +"death newer first": "décès plus récent" +"documents more first": "plus de documents" +"documents fewer first": "moins de documents" "%d years old": "%d ans" "BC": "av. J.-C." "Before Christ": "Avant Jésus-Christ" diff --git a/internal/webserver/embedded/translations/ru.yml b/internal/webserver/embedded/translations/ru.yml index 43643f74..10c2a1a7 100644 --- a/internal/webserver/embedded/translations/ru.yml +++ b/internal/webserver/embedded/translations/ru.yml @@ -1,7 +1,12 @@ "_language": "Русский" "Search": "Поиск" +"Documents": "Документы" +"Authors": "Авторы" "Search results": "Результаты поиска" "Search in %d documents": "Поиск по %d документам" +"Search in %d authors": "Поиск по %d авторам" +"Search in %d documents and %d authors": "Поиск по %d документам и %d авторам" +"No results found": "Результаты не найдены" "Download": "Скачать" "%d documents found": "Найдено %d документов" "No documents found": "Документы не найдены" @@ -118,6 +123,7 @@ "Completed": "Прочитанное" "Completions": "Прочитанное" "documents": "документов" +"document": "документ" "words": "слов" "All time": "Всё время" "%s's completed documents": "Прочитанные документы: %s" @@ -141,6 +147,9 @@ "An error occurred while uploading the image": "Произошла ошибка при загрузке изображения" "Click to upload a custom image": "Нажмите, чтобы загрузить пользовательское изображение" "Indexing in progress, search results may not be accurate.": "Идёт индексация; результаты поиска могут быть неточными." +"Retrieving author information, author pages may not be complete yet.": "Получение информации об авторах; страницы авторов могут быть неполными." +"Author information progress": "Прогресс получения информации об авторах" +"Indexing progress": "Прогресс индексации" "Remaining time: %s minutes": "Оставшееся время: %s мин." "There was an error deleting the user, please try again later": "Ошибка при удалении пользователя. Повторите попытку позже." "A user with this username already exists": "Пользователь с таким именем уже существует" @@ -165,6 +174,26 @@ "Never": "Никогда" "Date of birth": "Дата рождения" "Date of death": "Дата смерти" +"Search authors": "Поиск авторов" +"No authors found": "Авторы не найдены" +"%d authors found": "Найдено авторов: %d" +"Name": "Имя" +"Gender": "Пол" +"All genders": "Любой пол" +"Male": "Мужской" +"Female": "Женский" +"Intersex": "Интерсекс" +"Transgender female": "Трансгендерная женщина" +"Transgender male": "Трансгендерный мужчина" +"Unknown": "Неизвестно" +"name A-Z": "имя А–Я" +"name Z-A": "имя Я–А" +"birth older first": "рождение: старые" +"birth newer first": "рождение: новые" +"death older first": "смерть: старые" +"death newer first": "смерть: новые" +"documents more first": "больше документов" +"documents fewer first": "меньше документов" "%d years old": "%d лет" "BC": "до н. э." "Before Christ": "До Рождества Христова" diff --git a/internal/webserver/embedded/views/layout.html b/internal/webserver/embedded/views/layout.html index 6428ab42..2cb35e88 100644 --- a/internal/webserver/embedded/views/layout.html +++ b/internal/webserver/embedded/views/layout.html @@ -27,7 +27,7 @@ {{template "partials/navbar" .}}
- {{template "partials/main" dict "Lang" .Lang "IndexingInProgress" .IndexingInProgress "RemainingIndexingTime" .RemainingIndexingTime "IndexingProgressPercentage" .IndexingProgressPercentage "Error" .Error "Warning" .Warning "Success" .Success "Embed" .Embed}} + {{template "partials/main" dict "Lang" .Lang "IndexingInProgress" .IndexingInProgress "IndexingProgressKind" .IndexingProgressKind "RemainingIndexingTime" .RemainingIndexingTime "IndexingProgressPercentage" .IndexingProgressPercentage "Error" .Error "Warning" .Warning "Success" .Success "Embed" .Embed}}
{{template "partials/share-modal" .}} {{template "partials/keyboard-shortcuts-modal" .}} diff --git a/internal/webserver/embedded/views/partials/author-search-filters.html b/internal/webserver/embedded/views/partials/author-search-filters.html new file mode 100644 index 00000000..40e31a72 --- /dev/null +++ b/internal/webserver/embedded/views/partials/author-search-filters.html @@ -0,0 +1,75 @@ +{{$idPrefix := or .FilterIdPrefix ""}} +
+ {{if $idPrefix}} +
+ {{t .Lang "Search"}} +
+
+ {{$name := ""}} + {{if .SearchFields}} + {{$name = .SearchFields.Name}} + {{end}} + + +
+
+
+ {{end}} +
+ {{t .Lang "Gender"}} +
+
+ {{$gender := or .SelectedGender ""}} + +
+
+
+
+ {{t .Lang "Date of birth"}} +
+
+ {{$birthDateFrom := ""}} + {{if .SearchFields}}{{$birthDateFrom = .SearchFields.BirthDateFrom}}{{end}} + {{template "partials/date-control" dict "Name" "birthdate-from" "Lang" .Lang "Date" $birthDateFrom "Label" "From" "IdPrefix" $idPrefix }} +
+
+ {{$birthDateTo := ""}} + {{if .SearchFields}}{{$birthDateTo = .SearchFields.BirthDateTo}}{{end}} + {{template "partials/date-control" dict "Name" "birthdate-to" "Lang" .Lang "Date" $birthDateTo "Label" "To" "IdPrefix" $idPrefix }} +
+
+
+
+ {{t .Lang "Date of death"}} +
+
+ {{$deathDateFrom := ""}} + {{if .SearchFields}}{{$deathDateFrom = .SearchFields.DeathDateFrom}}{{end}} + {{template "partials/date-control" dict "Name" "deathdate-from" "Lang" .Lang "Date" $deathDateFrom "Label" "From" "IdPrefix" $idPrefix }} +
+
+ {{$deathDateTo := ""}} + {{if .SearchFields}}{{$deathDateTo = .SearchFields.DeathDateTo}}{{end}} + {{template "partials/date-control" dict "Name" "deathdate-to" "Lang" .Lang "Date" $deathDateTo "Label" "To" "IdPrefix" $idPrefix }} +
+
+
+
+
+ +
+
+
+ +{{if and (or $idPrefix (not .AuthorsSearchPage)) (not .HomeSearch)}} + + +{{end}} diff --git a/internal/webserver/embedded/views/partials/authors-list-content.html b/internal/webserver/embedded/views/partials/authors-list-content.html new file mode 100644 index 00000000..b3ffd14a --- /dev/null +++ b/internal/webserver/embedded/views/partials/authors-list-content.html @@ -0,0 +1,13 @@ +{{if gt .Results.TotalHits 0}} +
    + {{range $i, $author := .Results.Hits}} +
  • + {{template "partials/authors-list-item" dict "Lang" $.Lang "Author" $author "Version" $.Version "DocumentCount" (index $.DocumentCounts $author.Slug)}} +
    • + {{end}} +
+{{end}} + +{{ $length := len .Paginator.Pages }} {{ if gt $length 1 }} +{{template "partials/pagination" .}} +{{end}} diff --git a/internal/webserver/embedded/views/partials/authors-list-fragments.html b/internal/webserver/embedded/views/partials/authors-list-fragments.html new file mode 100644 index 00000000..c25b59f7 --- /dev/null +++ b/internal/webserver/embedded/views/partials/authors-list-fragments.html @@ -0,0 +1,2 @@ +
{{template "partials/authors-list-header" .}}
+
{{template "partials/authors-list-content" .}}
diff --git a/internal/webserver/embedded/views/partials/authors-list-header.html b/internal/webserver/embedded/views/partials/authors-list-header.html new file mode 100644 index 00000000..fe08afcb --- /dev/null +++ b/internal/webserver/embedded/views/partials/authors-list-header.html @@ -0,0 +1,15 @@ +{{$lang := .Lang}} +
+ {{if eq .Results.TotalHits 0}} +
+

{{t .Lang "No authors found"}}

+
+ {{else}} +
+

{{t .Lang "%d authors found" .Results.TotalHits }}

+
+
+ {{template "partials/sort" dict "SortURL" .SortURL "SortBy" .SortBy "Lang" $lang "AdditionalSortOptions" .AdditionalSortOptions}} +
+ {{end}} +
diff --git a/internal/webserver/embedded/views/partials/authors-list-item.html b/internal/webserver/embedded/views/partials/authors-list-item.html new file mode 100644 index 00000000..5eaf5d02 --- /dev/null +++ b/internal/webserver/embedded/views/partials/authors-list-item.html @@ -0,0 +1,29 @@ +
+
+ +
+ {{.Author.Name}} +
+ +
+
+

{{.Author.Name}}

+ {{if and (ne .Author.BirthName "") (not .Author.BirthNameIncludesName)}} +

{{.Author.BirthName}}

+ {{end}} + {{if index .Author.Description .Lang}} +

{{index .Author.Description .Lang}}

+ {{end}} +
+ {{if ne .Author.DateOfBirth.Date 0}} +
{{t .Lang "Date of birth"}}
+
+ {{end}} + {{if ne .Author.DateOfDeath.Date 0}} +
{{t .Lang "Date of death"}}
+
+ {{end}} +
+

{{.DocumentCount}} {{if eq .DocumentCount 1}}{{t .Lang "document"}}{{else}}{{t .Lang "documents"}}{{end}}

+
+
diff --git a/internal/webserver/embedded/views/partials/search-filters.html b/internal/webserver/embedded/views/partials/document-search-filters.html similarity index 95% rename from internal/webserver/embedded/views/partials/search-filters.html rename to internal/webserver/embedded/views/partials/document-search-filters.html index e3b2a07b..4b5bbb98 100644 --- a/internal/webserver/embedded/views/partials/search-filters.html +++ b/internal/webserver/embedded/views/partials/document-search-filters.html @@ -1,5 +1,5 @@ {{$idPrefix := or .FilterIdPrefix ""}} -
+
{{if $idPrefix}}
{{t .Lang "Search"}} @@ -109,20 +109,18 @@
- {{if not .DocumentsSearchPage}} -
+
- {{end}}
{{/* Include script and i18n from sidebar when on documents search page (so script runs after sidebar is in DOM); otherwise include from offcanvas. */}} -{{if or $idPrefix (not .DocumentsSearchPage)}} +{{if and (or $idPrefix (not .DocumentsSearchPage)) (not .HomeSearch)}} - + {{end}} diff --git a/internal/webserver/embedded/views/partials/main.html b/internal/webserver/embedded/views/partials/main.html index 0b3eb580..7fd1f8a8 100644 --- a/internal/webserver/embedded/views/partials/main.html +++ b/internal/webserver/embedded/views/partials/main.html @@ -2,8 +2,13 @@ {{if .IndexingInProgress}}