From 2e9f616ef298c4eaf3bdf553af1c7cb7622a8e3d Mon Sep 17 00:00:00 2001 From: Chris Korhonen Date: Wed, 7 Jan 2026 18:37:39 -0500 Subject: [PATCH 01/79] Fix #124: Range with Volumes problem --- internal/flag/parser/range.go | 14 +++-- internal/flag/parser/range_test.go | 5 ++ pkg/sites/loader.go | 40 ++++++++++++--- pkg/sites/loader_test.go | 82 ++++++++++++++++++++++++++++++ 4 files changed, 131 insertions(+), 10 deletions(-) diff --git a/internal/flag/parser/range.go b/internal/flag/parser/range.go index 165769d1..23e9aa66 100644 --- a/internal/flag/parser/range.go +++ b/internal/flag/parser/range.go @@ -8,19 +8,20 @@ import ( ) // ParseIssuesRange the range of issues. -// Format [start-end]. +// Format [start-end] or [volume.issue-volume.issue]. +// Examples: "1-5", "3.1-9.5", "4.78-4.99" (volume 4, issues 78-99) func ParseIssuesRange(rng string) (float64, float64, error) { values := strings.Split(rng, "-") if len(values) != 2 { return 0, 0, errors.New("wrong range format") } - startRange, err := strconv.ParseFloat(values[0], 64) + startRange, err := parseVolumeIssue(values[0]) if err != nil { return 0, 0, fmt.Errorf("wrong the start range value: %v", err) } - endRange, err := strconv.ParseFloat(values[1], 64) + endRange, err := parseVolumeIssue(values[1]) if err != nil { return 0, 0, fmt.Errorf("wrong the end range value: %v", err) } @@ -35,3 +36,10 @@ func ParseIssuesRange(rng string) (float64, float64, error) { return startRange, endRange, nil } + +// parseVolumeIssue parses a value that may be in format "volume.issue" or just a simple number. +// For backwards compatibility and simplicity, it just parses as a regular float. +// Examples: "4.78" -> 4.78, "4.099" -> 4.099, "3.1" -> 3.1, "5" -> 5.0 +func parseVolumeIssue(value string) (float64, error) { + return strconv.ParseFloat(value, 64) +} diff --git a/internal/flag/parser/range_test.go b/internal/flag/parser/range_test.go index 479758cf..515b3130 100644 --- a/internal/flag/parser/range_test.go +++ b/internal/flag/parser/range_test.go @@ -25,6 +25,11 @@ func TestParseIssuesRange(t *testing.T) { {"2-1", 0, 0, true}, {"1", 0, 0, true}, {"wrong range", 0, 0, true}, + // Volume.issue format tests + {"4.78-4.99", 4.78, 4.99, false}, // Nightwing V4 #078-099 + {"4.078-4.099", 4.078, 4.099, false}, // Same with leading zeros + {"1.10-1.20", 1.10, 1.20, false}, // Volume 1, issues 10-20 + {"2.01-2.05", 2.01, 2.05, false}, // Volume 2, issues 01-05 } for _, tc := range tt { diff --git a/pkg/sites/loader.go b/pkg/sites/loader.go index 4125853b..3a8c0926 100644 --- a/pkg/sites/loader.go +++ b/pkg/sites/loader.go @@ -65,24 +65,50 @@ func initializeCollection(issues []string, options *config.Options, base BaseSit return collection, nil } -var onlyNumbers = regexp.MustCompile("[^0-9]+[^.][^0-9]+") +var volumeAndIssuePattern = regexp.MustCompile(`v(\d+)[-_]0*(\d+)`) +var onlyDigits = regexp.MustCompile(`\d+`) func notInIssuesRange(issueNumber string, start, end float64) bool { if start == 0 || end == 0 { return false } - normalizedNumber := onlyNumbers.ReplaceAllString(issueNumber, "") - if normalizedNumber == "" { + number := extractIssueNumberForRange(issueNumber) + if number == 0 { return true } - number, err := strconv.ParseFloat(normalizedNumber, 64) - if err != nil { - return true + return number < start || number > end +} + +// extractIssueNumberForRange extracts a numeric value from an issue number string +// for range comparison. It supports: +// 1. Volume.Issue format where v- becomes . as a decimal +// (e.g., "v4-078" -> 4.78 for Volume 4, Issue 78) +// 2. Simple numeric format (e.g., "078" -> 78, "20.5" -> 20.5) +func extractIssueNumberForRange(issueNumber string) float64 { + // Try to match volume and issue pattern (e.g., "v4-078-2016") + if matches := volumeAndIssuePattern.FindStringSubmatch(issueNumber); matches != nil { + volume, _ := strconv.Atoi(matches[1]) + issue, _ := strconv.Atoi(matches[2]) + // Combine as volume.issue decimal (e.g., volume 4, issue 78 becomes 4.78) + // This allows users to specify ranges like "4.78-4.99" for V4 issues 78-99 + return float64(volume) + float64(issue)/100.0 } - return number < start || number > end + // Try to parse as a simple float (e.g., "20.5") + if number, err := strconv.ParseFloat(issueNumber, 64); err == nil { + return number + } + + // Extract the first sequence of digits (e.g., "078" from "078-something") + if matches := onlyDigits.FindString(issueNumber); matches != "" { + if number, err := strconv.ParseFloat(matches, 64); err == nil { + return number + } + } + + return 0 } // LoadComicFromSource will return an `comic` instance initialized based on the source diff --git a/pkg/sites/loader_test.go b/pkg/sites/loader_test.go index 8ee3f9f3..2a13aa95 100644 --- a/pkg/sites/loader_test.go +++ b/pkg/sites/loader_test.go @@ -198,3 +198,85 @@ func TestFloatIssuesRange(t *testing.T) { }) } } + +func TestVolumeIssuesRange(t *testing.T) { + tt := []struct { + name string + input string + start float64 + end float64 + returnValue bool + }{ + // Volume 4, Issue 78-99 range tests (user specifies: -range=4.78-4.99) + {"v4-078 in range", "v4-078-2016", 4.78, 4.99, false}, + {"v4-099 in range", "v4-099-2016", 4.78, 4.99, false}, + {"v4-077 out of range (too low)", "v4-077-2016", 4.78, 4.99, true}, + {"v4-100 out of range (too high)", "v4-100-2016", 4.78, 4.99, true}, + {"v3-078 wrong volume", "v3-078-2016", 4.78, 4.99, true}, + {"v5-078 wrong volume", "v5-078-2016", 4.78, 4.99, true}, + + // Volume 2, Issue 1-50 range tests (user specifies: -range=2.01-2.50) + {"v2-001 in range", "v2-001-1989", 2.01, 2.50, false}, + {"v2-025 in range", "v2-025-1989", 2.01, 2.50, false}, + {"v2-050 in range", "v2-050-1989", 2.01, 2.50, false}, + {"v2-051 out of range", "v2-051-1989", 2.01, 2.50, true}, + + // Edge cases with different formats + {"v4-078 without year", "v4-078", 4.78, 4.99, false}, + {"v4_078 with underscore", "v4_078", 4.78, 4.99, false}, + {"v10-005 two-digit volume", "v10-005", 10.05, 10.10, false}, + + // Backwards compatibility with simple numeric issues + {"078 simple format", "078", 78, 99, false}, + {"100 simple format out of range", "100", 78, 99, true}, + {"issue-1 with prefix", "issue-1", 1, 3, false}, + {"issue-5 with prefix out of range", "issue-5", 1, 3, true}, + } + + for _, tc := range tt { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.returnValue, notInIssuesRange(tc.input, tc.start, tc.end), + "Issue %s with range %.2f-%.2f", tc.input, tc.start, tc.end) + }) + } +} + +func TestExtractIssueNumberForRange(t *testing.T) { + tt := []struct { + name string + input string + expected float64 + }{ + // Volume and issue format + {"v4-078-2016", "v4-078-2016", 4.78}, + {"v4-099-2016", "v4-099-2016", 4.99}, + {"v2-075-1989", "v2-075-1989", 2.75}, + {"v4-078 no year", "v4-078", 4.78}, + {"v4_078 underscore", "v4_078", 4.78}, + {"v10-005 two-digit volume", "v10-005", 10.05}, + + // Simple numeric format + {"078", "078", 78}, + {"99", "99", 99}, + {"1", "1", 1}, + + // Decimal format + {"20.5", "20.5", 20.5}, + {"3.14", "3.14", 3.14}, + + // With prefixes + {"issue-1", "issue-1", 1}, + {"issue-123", "issue-123", 123}, + + // Edge cases + {"empty", "", 0}, + {"no numbers", "abc", 0}, + } + + for _, tc := range tt { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.expected, extractIssueNumberForRange(tc.input), + "Issue %s should extract to %.2f", tc.input, tc.expected) + }) + } +} From 831de12c490f804c046665763d713118c7711993 Mon Sep 17 00:00:00 2001 From: hollisticated-horse Date: Mon, 20 Oct 2025 18:47:11 +0200 Subject: [PATCH 02/79] Introduce runner orchestration and version check improvements --- AGENTS.md | 16 +++ cmd/app/downloader.go | 168 +++++++++++++++++++++---------- cmd/app/runner_test.go | 47 +++++++++ cmd/downloader/main.go | 23 +++-- cmd/downloader/main_test.go | 108 ++++++++++++++++++++ go.mod | 2 - internal/version/version.go | 118 ++++++++++++++++++---- internal/version/version_test.go | 121 ++++++++++++++++++++++ pkg/http/client.go | 8 ++ 9 files changed, 528 insertions(+), 83 deletions(-) create mode 100644 AGENTS.md create mode 100644 cmd/app/runner_test.go create mode 100644 cmd/downloader/main_test.go create mode 100644 internal/version/version_test.go diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 00000000..ee856cc7 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,16 @@ +# Repository Guidelines + +## Project Structure & Module Organization +The project targets Go 1.23+ with a standard layout. `cmd/downloader` hosts the CLI entrypoint, while `cmd/gui` wraps the GUI built with Fyne; both depend on reusable packages under `pkg/`. Key subpackages include `pkg/core` for orchestration, `pkg/sites` for site-specific scrapers, and `pkg/util` for image and filesystem helpers. Supporting flag parsing and other internals live in `internal/`. Docs for contributors live in `docs/`, assets in `img/`, and release artifacts appear under `build/` when generated. + +## Build, Test, and Development Commands +Run `go build -o ./bin/comics-downloader ./cmd/downloader` for a local CLI binary. The GUI can be built with `go build -o ./bin/comics-downloader-gui ./cmd/gui` once Fyne prerequisites are met. Cross-platform artifacts are automated via Make targets such as `make linux-x86-64-build` or the aggregate `make builds`. Use `fyne-cross windows -output comics-downloader-gui-windows.exe ./cmd/gui` when Docker-based cross-compilation is preferred. Always execute `go test -v ./...` before pushing; CI mirrors this command and reports coverage with Coveralls. + +## Coding Style & Naming Conventions +Format all Go files with `gofmt` (invoked via `go fmt ./...`) to preserve canonical tab-indented style and import ordering. Follow idiomatic Go naming: exported identifiers use CamelCase, unexported use camelCase, constants are ALL_CAPS only when enumerations require clarity. Keep packages small and cohesive; prefer descriptive filenames like `mangadex.go` aligning with the site handled. Log through the existing Logrus helpers to stay consistent with current output. + +## Testing Guidelines +Place tests alongside implementation in `*_test.go` files, using table-driven cases where practical. Run targeted suites with `go test ./pkg/sites` during feature work, and use `go test -cover ./...` to ensure coverage does not regress—Coveralls tracks the master branch. When scraping new sources, add integration-style tests that stub HTTP calls where possible to keep the suite deterministic. + +## Commit & Pull Request Guidelines +Commit subjects should be concise and imperative (`add`, `fix`, `update`), optionally prefixed with scopes like `fix:` as seen in history. Commit early and keep diffs focused; avoid mixing GUI and CLI changes in one commit. Pull requests must target `master`, include a brief summary of the change, reference related issues, and note how to reproduce or test the update (logs, commands, screenshots for GUI tweaks). Ensure tests pass locally before requesting review. diff --git a/cmd/app/downloader.go b/cmd/app/downloader.go index 39411709..39c618b3 100644 --- a/cmd/app/downloader.go +++ b/cmd/app/downloader.go @@ -1,6 +1,7 @@ package app import ( + "context" "fmt" "os" "strings" @@ -10,7 +11,7 @@ import ( "github.com/Girbons/comics-downloader/internal/version" "github.com/Girbons/comics-downloader/pkg/config" "github.com/Girbons/comics-downloader/pkg/detector" - "github.com/Girbons/comics-downloader/pkg/http" + httpclient "github.com/Girbons/comics-downloader/pkg/http" "github.com/Girbons/comics-downloader/pkg/sites" "github.com/sirupsen/logrus" ) @@ -22,110 +23,169 @@ var ( Messages = make(chan string) ) -func download(options *config.Options) { - if options.Debug { - options.Logger.SetLevel(logrus.DebugLevel) +// Runner orchestrates a download session based on immutable user input. +type Runner struct { + base config.Options + bindToChannel bool + messages chan string + + loggerFactory func(bind bool, messages chan string) *logger.Logger + clientFactory func() *httpclient.ComicClient + sleep func(time.Duration) +} + +// NewRunner returns a Runner with default factories. +func NewRunner(base config.Options) *Runner { + return &Runner{ + base: base, + loggerFactory: func(bind bool, messages chan string) *logger.Logger { + return logger.NewLogger(bind, messages) + }, + clientFactory: httpclient.NewComicClient, + sleep: time.Sleep, + } +} + +// WithChannelBinding enables GUI-friendly logging via the provided channel. +func (r *Runner) WithChannelBinding(messages chan string) { + r.bindToChannel = true + r.messages = messages +} + +func (r *Runner) prepareOptions() config.Options { + opts := r.base + if r.loggerFactory != nil { + opts.Logger = r.loggerFactory(r.bindToChannel, r.messages) + } + if r.clientFactory != nil { + opts.Client = r.clientFactory() + } + return opts +} + +// Run executes a download session, respecting daemon configuration. +func (r *Runner) Run() { + opts := r.prepareOptions() + if opts.Logger == nil { + opts.Logger = logger.NewLogger(false, nil) + } + if opts.Client == nil { + opts.Client = httpclient.NewComicClient() + } + + if opts.URL == "" { + opts.Logger.Error("url parameter is required") + return + } + + if opts.Debug { + opts.Logger.SetLevel(logrus.DebugLevel) + } + + // daemon is started only if `all` or `last` flags are used + if opts.Daemon && (opts.All || opts.Last) { + for { + r.download(opts) + r.sleep(time.Duration(opts.DaemonTimeout) * time.Second) + } } - if options.All && options.Last { - options.Last = false - options.Logger.Warning("all and last are selected, all parameter will be used") + r.download(opts) +} + +func (r *Runner) download(base config.Options) { + opts := base + + if opts.All && opts.Last { + opts.Last = false + opts.Logger.Warning("all and last are selected, all parameter will be used") } // enforce `all` flag when `range` is used. - if options.IssuesRange != "" && !options.All { - options.All = true + if opts.IssuesRange != "" && !opts.All { + opts.All = true } - if options.OutputFolder == "" { + outputFolder := opts.OutputFolder + if outputFolder == "" { dir, err := os.Getwd() if err != nil { - options.Logger.Error("Error determining current directory: %v\n") - options.OutputFolder = "." + opts.Logger.Error(fmt.Sprintf("Error determining current directory: %v", err)) + outputFolder = "." } else { - options.OutputFolder = dir + outputFolder = dir } } - isNewVersionAvailable, newVersionLink, err := version.IsNewAvailable() + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + + isNewVersionAvailable, newVersionLink, err := version.IsNewAvailable(ctx, opts.Client.HTTPClient()) if err != nil { - options.Logger.Error("There was an error while checking for a new comics-downloader version") + opts.Logger.Error("There was an error while checking for a new comics-downloader version") } if isNewVersionAvailable { - options.Logger.Info(fmt.Sprintf("A new comics-downloader version is available at %s", newVersionLink)) + opts.Logger.Info(fmt.Sprintf("A new comics-downloader version is available at %s", newVersionLink)) } - urls := options.URL - - for _, u := range strings.Split(urls, ",") { - if u == "" { + for _, rawURL := range strings.Split(opts.URL, ",") { + trimmedURL := strings.TrimSpace(rawURL) + if trimmedURL == "" { continue } + perURL := opts + perURL.URL = trimmedURL + perURL.OutputFolder = outputFolder + // check if the link is supported - source, check, isDisabled := detector.DetectComic(u) + source, check, isDisabled := detector.DetectComic(trimmedURL) - options.Source = source - options.URL = u + perURL.Source = source if !check { - options.Logger.Error("This site is not supported") + perURL.Logger.Error("This site is not supported") continue } if isDisabled { - options.Logger.Warning("Site currently disabled, please check https://github.com/Girbons/comics-downloader/issues/") + perURL.Logger.Warning("Site currently disabled, please check https://github.com/Girbons/comics-downloader/issues/") continue } - options.Logger.Info("Downloading...") - collection, err := sites.LoadComicFromSource(options) + perURL.Logger.Info("Downloading...") + collection, err := sites.LoadComicFromSource(&perURL) if err != nil { - options.Logger.Error(err.Error()) + perURL.Logger.Error(err.Error()) continue } for _, comic := range collection { - if options.ImagesOnly { - _, err = comic.DownloadImages(options) + if perURL.ImagesOnly { + _, err = comic.DownloadImages(&perURL) } else { - err = comic.MakeComic(options) + err = comic.MakeComic(&perURL) } if err != nil { - options.Logger.Error(err.Error()) + perURL.Logger.Error(err.Error()) } } } } -// GuiRun will start the GUI app +// GuiRun will start the GUI app. func GuiRun(options *config.Options) { AppStatus <- true - options.Logger = logger.NewLogger(true, Messages) - download(options) + runner := NewRunner(*options) + runner.WithChannelBinding(Messages) + runner.Run() AppStatus <- false } -// Run will start the CLI app +// Run will start the CLI app. func Run(options *config.Options) { - options.Logger = logger.NewLogger(false, Messages) - options.Client = http.NewComicClient() - - // link is required - if options.URL == "" { - options.Logger.Error("url parameter is required") - return - } - - // daemon is started only if `all` or `last` flags are used - if options.Daemon && (options.All || options.Last) { - for { - download(options) - time.Sleep(time.Duration(options.DaemonTimeout) * time.Second) - } - } - - download(options) + runner := NewRunner(*options) + runner.Run() } diff --git a/cmd/app/runner_test.go b/cmd/app/runner_test.go new file mode 100644 index 00000000..9797803d --- /dev/null +++ b/cmd/app/runner_test.go @@ -0,0 +1,47 @@ +package app + +import ( + "strings" + "testing" + + "github.com/Girbons/comics-downloader/pkg/config" +) + +func TestRunnerPrepareOptionsProvidesDependencies(t *testing.T) { + runner := NewRunner(config.Options{}) + + opts := runner.prepareOptions() + + if opts.Logger == nil { + t.Fatalf("expected logger to be initialized") + } + + if opts.Client == nil { + t.Fatalf("expected http client to be initialized") + } + + if runner.base.Logger != nil { + t.Fatalf("expected runner base logger to remain nil") + } + if runner.base.Client != nil { + t.Fatalf("expected runner base client to remain nil") + } +} + +func TestRunnerRunRequiresURL(t *testing.T) { + msgs := make(chan string, 1) + + runner := NewRunner(config.Options{}) + runner.WithChannelBinding(msgs) + + runner.Run() + + select { + case msg := <-msgs: + if !strings.Contains(msg, "url parameter is required") { + t.Fatalf("expected error message about missing url, got %q", msg) + } + default: + t.Fatalf("expected an error message to be sent") + } +} diff --git a/cmd/downloader/main.go b/cmd/downloader/main.go index 6e6af4c6..e4930666 100644 --- a/cmd/downloader/main.go +++ b/cmd/downloader/main.go @@ -66,15 +66,8 @@ func init() { flag.IntVar(&daemonTimeout, "daemon-timeout", 600, "DaemonTimeout (seconds), specifies how often the downloader runs") } -func main() { - flag.Parse() - - if versionFlag { - fmt.Println("comics-downloader version", version.Tag) - os.Exit(0) - } - - options := &config.Options{ +func buildOptions() config.Options { + return config.Options{ Debug: debug, All: all, Last: last, @@ -93,6 +86,16 @@ func main() { IssuesRange: issuesRange, IssueFolderName: issueFolderName, } +} + +func main() { + flag.Parse() + + if versionFlag { + fmt.Println("comics-downloader version", version.Tag) + os.Exit(0) + } - app.Run(options) + opts := buildOptions() + app.Run(&opts) } diff --git a/cmd/downloader/main_test.go b/cmd/downloader/main_test.go new file mode 100644 index 00000000..db0e04e3 --- /dev/null +++ b/cmd/downloader/main_test.go @@ -0,0 +1,108 @@ +package main + +import ( + "testing" +) + +func TestBuildOptionsCopiesGlobals(t *testing.T) { + prev := struct { + debug bool + all bool + last bool + imagesOnly bool + imagesFormat string + country string + forceAspect bool + format string + customComicName string + issueNumberNameOnly bool + url string + outputFolder string + createDefaultPath bool + daemon bool + daemonTimeout int + issuesRange string + issueFolderName string + }{ + debug: debug, + all: all, + last: last, + imagesOnly: imagesOnly, + imagesFormat: imagesFormat, + country: country, + forceAspect: forceAspect, + format: format, + customComicName: customComicName, + issueNumberNameOnly: issueNumberNameOnly, + url: url, + outputFolder: outputFolder, + createDefaultPath: createDefaultPath, + daemon: daemon, + daemonTimeout: daemonTimeout, + issuesRange: issuesRange, + issueFolderName: issueFolderName, + } + defer func() { + debug = prev.debug + all = prev.all + last = prev.last + imagesOnly = prev.imagesOnly + imagesFormat = prev.imagesFormat + country = prev.country + forceAspect = prev.forceAspect + format = prev.format + customComicName = prev.customComicName + issueNumberNameOnly = prev.issueNumberNameOnly + url = prev.url + outputFolder = prev.outputFolder + createDefaultPath = prev.createDefaultPath + daemon = prev.daemon + daemonTimeout = prev.daemonTimeout + issuesRange = prev.issuesRange + issueFolderName = prev.issueFolderName + }() + + debug = true + all = true + last = true + imagesOnly = true + imagesFormat = "png" + country = "jp" + forceAspect = true + format = "epub" + customComicName = "custom" + issueNumberNameOnly = true + url = "http://example.com/comic" + outputFolder = "/tmp/output" + createDefaultPath = false + daemon = true + daemonTimeout = 42 + issuesRange = "1-5" + issueFolderName = "chapter-" + + opts := buildOptions() + + if !opts.Debug || !opts.All || !opts.Last || !opts.ImagesOnly { + t.Fatalf("expected boolean flags to be copied into options: %+v", opts) + } + + if opts.ImagesFormat != "png" || opts.Country != "jp" || opts.Format != "epub" { + t.Fatalf("expected string values to be copied, got %+v", opts) + } + + if opts.CustomComicName != "custom" || opts.URL != "http://example.com/comic" { + t.Fatalf("expected URL and custom name to be copied, got %+v", opts) + } + + if opts.OutputFolder != "/tmp/output" || opts.IssueFolderName != "chapter-" { + t.Fatalf("expected folder values to be copied, got %+v", opts) + } + + if opts.DaemonTimeout != 42 || !opts.Daemon || opts.CreateDefaultPath { + t.Fatalf("expected daemon configuration to be copied, got %+v", opts) + } + + if opts.IssuesRange != "1-5" { + t.Fatalf("expected issues range to be copied, got %q", opts.IssuesRange) + } +} diff --git a/go.mod b/go.mod index b06c0ac9..4b114139 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,6 @@ require ( github.com/anaskhan96/soup v1.2.5 github.com/bmaupin/go-epub v1.1.0 github.com/dlclark/regexp2 v1.10.0 - github.com/google/go-github v17.0.0+incompatible github.com/jung-kurt/gofpdf v1.16.2 github.com/mholt/archives v0.1.2 github.com/schollz/progressbar/v2 v2.15.0 @@ -37,7 +36,6 @@ require ( github.com/gofrs/uuid v3.1.0+incompatible // indirect github.com/goki/freetype v0.0.0-20181231101311-fa8a33aabaff // indirect github.com/google/go-cmp v0.5.9 // indirect - github.com/google/go-querystring v1.1.0 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect diff --git a/internal/version/version.go b/internal/version/version.go index 9baead24..78795aaf 100644 --- a/internal/version/version.go +++ b/internal/version/version.go @@ -2,33 +2,117 @@ package version import ( "context" + "encoding/json" + "fmt" + "net/http" + "sync" + "time" - "github.com/google/go-github/github" "golang.org/x/mod/semver" ) -// Tag specifies the current release tag. -// It needs to be manually updated. -const Tag = "v0.34-alpha" +const ( + releasesEndpoint = "https://api.github.com/repos/Girbons/comics-downloader/releases?per_page=1" + cacheTTL = time.Minute * 15 +) + +// Tag specifies the current release tag. Overridden at build time via -ldflags. +var Tag = "development" + +type release struct { + TagName string `json:"tag_name"` + HTMLURL string `json:"html_url"` +} + +type releaseResult struct { + checked time.Time + available bool + link string + err error +} + +var ( + cacheMu sync.Mutex + cachedResult releaseResult +) + +// IsNewAvailable fetches the latest release information and compares it against the current Tag. +// Results are cached for a short period to avoid repeatedly hammering the API. +func IsNewAvailable(ctx context.Context, client *http.Client) (bool, string, error) { + if ctx == nil { + ctx = context.Background() + } + if client == nil { + client = http.DefaultClient + } + + cacheMu.Lock() + if !cachedResult.checked.IsZero() && time.Since(cachedResult.checked) < cacheTTL { + res := cachedResult + cacheMu.Unlock() + return res.available, res.link, res.err + } + cacheMu.Unlock() + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, releasesEndpoint, nil) + if err != nil { + return false, "", err + } -// IsNewAvailable will fetch the latest project releases -// and will compare the latest release Tag against the current Tag. -func IsNewAvailable() (bool, string, error) { - ctx := context.Background() - client := github.NewClient(nil) - releases, _, err := client.Repositories.ListReleases(ctx, "Girbons", "comics-downloader", nil) + resp, err := client.Do(req) if err != nil { + updateCache(false, "", err) + return false, "", err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + err = fmt.Errorf("unexpected status code: %d", resp.StatusCode) + updateCache(false, "", err) + return false, "", err + } + + var releases []release + if err := json.NewDecoder(resp.Body).Decode(&releases); err != nil { + updateCache(false, "", err) + return false, "", err + } + if len(releases) == 0 { + err = fmt.Errorf("no releases found") + updateCache(false, "", err) + return false, "", err + } + latest := releases[0] + + if !semver.IsValid(Tag) || !semver.IsValid(latest.TagName) { + err = fmt.Errorf("invalid semver tag (current=%q latest=%q)", Tag, latest.TagName) + updateCache(false, "", err) return false, "", err } - // Compare returns an integer comparing two versions - // according to semantic version precedence. - result := semver.Compare(Tag, *releases[0].TagName) + if semver.Compare(Tag, latest.TagName) < 0 { + updateCache(true, latest.HTMLURL, nil) + return true, latest.HTMLURL, nil + } + + updateCache(false, "", nil) + return false, "", nil +} - // -1 if v < w - if result == -1 { - return true, *releases[0].HTMLURL, err +func updateCache(available bool, link string, err error) { + cacheMu.Lock() + defer cacheMu.Unlock() + cachedResult = releaseResult{ + checked: time.Now(), + available: available, + link: link, + err: err, } +} - return false, "", err +// ResetCache clears the cached release lookup result. Intended for testing. +func ResetCache() { + cacheMu.Lock() + defer cacheMu.Unlock() + cachedResult = releaseResult{} } diff --git a/internal/version/version_test.go b/internal/version/version_test.go new file mode 100644 index 00000000..ef2ce6d2 --- /dev/null +++ b/internal/version/version_test.go @@ -0,0 +1,121 @@ +package version + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "net/url" + "sync/atomic" + "testing" + "time" +) + +type roundTripFunc func(*http.Request) (*http.Response, error) + +func (f roundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) { + return f(req) +} + +func testHTTPClient(target *url.URL) *http.Client { + return &http.Client{ + Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) { + cloned := req.Clone(req.Context()) + cloned.URL.Scheme = target.Scheme + cloned.URL.Host = target.Host + cloned.Host = target.Host + return http.DefaultTransport.RoundTrip(cloned) + }), + Timeout: time.Second * 5, + } +} + +func TestIsNewAvailableCachesResult(t *testing.T) { + ResetCache() + originalTag := Tag + Tag = "v0.1.0" + defer func() { + Tag = originalTag + ResetCache() + }() + + var hits int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + atomic.AddInt32(&hits, 1) + w.Header().Set("Content-Type", "application/json") + fmt.Fprint(w, `[{"tag_name":"v0.2.0","html_url":"http://example.com/latest"}]`) + })) + defer server.Close() + + serverURL, err := url.Parse(server.URL) + if err != nil { + t.Fatalf("failed to parse test server url: %v", err) + } + + client := testHTTPClient(serverURL) + + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + + available, link, err := IsNewAvailable(ctx, client) + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + if !available { + t.Fatalf("expected new version to be available") + } + if link != "http://example.com/latest" { + t.Fatalf("unexpected link: %s", link) + } + + if got := atomic.LoadInt32(&hits); got != 1 { + t.Fatalf("expected single HTTP call, got %d", got) + } + + ctx2, cancel2 := context.WithTimeout(context.Background(), time.Second) + defer cancel2() + available, _, err = IsNewAvailable(ctx2, client) + if err != nil { + t.Fatalf("expected cached call to succeed, got %v", err) + } + if !available { + t.Fatalf("expected cached call to preserve availability") + } + if got := atomic.LoadInt32(&hits); got != 1 { + t.Fatalf("expected cached call to avoid HTTP, got %d requests", got) + } +} + +func TestIsNewAvailableInvalidSemver(t *testing.T) { + ResetCache() + originalTag := Tag + Tag = "not-a-semver" + defer func() { + Tag = originalTag + ResetCache() + }() + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + fmt.Fprint(w, `[{"tag_name":"v0.2.0","html_url":"http://example.com/latest"}]`) + })) + defer server.Close() + + serverURL, err := url.Parse(server.URL) + if err != nil { + t.Fatalf("failed to parse test server url: %v", err) + } + + client := testHTTPClient(serverURL) + + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + + available, _, err := IsNewAvailable(ctx, client) + if err == nil { + t.Fatalf("expected error for invalid semver") + } + if available { + t.Fatalf("expected availability to be false on error") + } +} diff --git a/pkg/http/client.go b/pkg/http/client.go index acb8e009..70a54f0c 100644 --- a/pkg/http/client.go +++ b/pkg/http/client.go @@ -17,6 +17,14 @@ func NewComicClient() *ComicClient { } } +// HTTPClient exposes the underlying http.Client instance. +func (c *ComicClient) HTTPClient() *http.Client { + if c == nil { + return nil + } + return c.Client +} + // PrepareRequest setup a `GET` request with customs headers. func (c *ComicClient) PrepareRequest(link, hostname string) (*http.Request, error) { req, err := http.NewRequest("GET", link, nil) From c106e6aa33ed92fb789855776ffe82c82961d3d6 Mon Sep 17 00:00:00 2001 From: hollisticated-horse Date: Mon, 20 Oct 2025 21:51:17 +0200 Subject: [PATCH 03/79] Enhance HTTP client with retries and user-agent handling --- cmd/app/downloader.go | 8 +- pkg/http/client.go | 167 ++++++++++++++++++++++++++++++++++++---- pkg/http/client_test.go | 76 ++++++++++++++++-- 3 files changed, 229 insertions(+), 22 deletions(-) diff --git a/cmd/app/downloader.go b/cmd/app/downloader.go index 39c618b3..a4fd3a85 100644 --- a/cmd/app/downloader.go +++ b/cmd/app/downloader.go @@ -41,8 +41,12 @@ func NewRunner(base config.Options) *Runner { loggerFactory: func(bind bool, messages chan string) *logger.Logger { return logger.NewLogger(bind, messages) }, - clientFactory: httpclient.NewComicClient, - sleep: time.Sleep, + clientFactory: func() *httpclient.ComicClient { + return httpclient.NewComicClient( + httpclient.WithUserAgent(fmt.Sprintf("comics-downloader/%s", version.Tag)), + ) + }, + sleep: time.Sleep, } } diff --git a/pkg/http/client.go b/pkg/http/client.go index 70a54f0c..f2d02c42 100644 --- a/pkg/http/client.go +++ b/pkg/http/client.go @@ -1,20 +1,95 @@ package http import ( + "context" + "errors" + "fmt" "net/http" "strings" + "time" ) -// ComicClient is the custom client. +const ( + defaultTimeout = 15 * time.Second + defaultRetryCount = 2 + defaultRetryWait = 500 * time.Millisecond + defaultUserAgent = "comics-downloader-client" +) + +// RateLimiter exposes a minimal interface for throttling outgoing requests. +type RateLimiter interface { + Wait(context.Context) error +} + +// Option modifies ComicClient behaviour. +type Option func(*ComicClient) + +// WithHTTPClient allows supplying a custom http.Client. +func WithHTTPClient(client *http.Client) Option { + return func(cc *ComicClient) { + if client != nil { + cc.client = client + } + } +} + +// WithRetry configures retry attempts and wait duration between retries. +func WithRetry(count int, wait time.Duration) Option { + return func(cc *ComicClient) { + if count >= 0 { + cc.retryCount = count + } + if wait >= 0 { + cc.retryWait = wait + } + } +} + +// WithRateLimiter sets a rate limiter to gate outbound requests. +func WithRateLimiter(limiter RateLimiter) Option { + return func(cc *ComicClient) { + cc.rateLimiter = limiter + } +} + +// WithUserAgent overrides the default user-agent header. +func WithUserAgent(agent string) Option { + return func(cc *ComicClient) { + if strings.TrimSpace(agent) != "" { + cc.userAgent = agent + } + } +} + +// ComicClient is the custom HTTP helper used across the downloader. type ComicClient struct { - Client *http.Client + client *http.Client + retryCount int + retryWait time.Duration + rateLimiter RateLimiter + userAgent string } -// NewComicClient returns a ComicClient instance. -func NewComicClient() *ComicClient { - return &ComicClient{ - Client: &http.Client{}, +// NewComicClient returns a ComicClient instance with sane defaults. +func NewComicClient(options ...Option) *ComicClient { + cc := &ComicClient{ + client: &http.Client{ + Timeout: defaultTimeout, + }, + retryCount: defaultRetryCount, + retryWait: defaultRetryWait, + userAgent: defaultUserAgent, + } + + for _, opt := range options { + opt(cc) + } + + if cc.client.Timeout == 0 { + cc.client.Timeout = defaultTimeout } + + return cc } // HTTPClient exposes the underlying http.Client instance. @@ -22,27 +97,91 @@ func (c *ComicClient) HTTPClient() *http.Client { if c == nil { return nil } - return c.Client + return c.client } -// PrepareRequest setup a `GET` request with customs headers. +// PrepareRequest setup a `GET` request with custom headers. func (c *ComicClient) PrepareRequest(link, hostname string) (*http.Request, error) { - req, err := http.NewRequest("GET", link, nil) + req, err := http.NewRequest(http.MethodGet, link, nil) + if err != nil { + return nil, err + } if strings.Contains(hostname, "manganato") || strings.Contains(hostname, "mangakakalot") { - req.Header.Add("Referer", link) + req.Header.Set("Referer", link) + } + + if c != nil && c.userAgent != "" { + req.Header.Set("User-Agent", c.userAgent) } - return req, err + return req, nil } -// GET Performs a Get request.. +// Do executes an HTTP request applying retry, timeout, and rate limiting policies. +func (c *ComicClient) Do(req *http.Request) (*http.Response, error) { + if c == nil { + return nil, errors.New("comic client is nil") + } + if req == nil { + return nil, errors.New("request is nil") + } + + attempts := c.retryCount + 1 + if attempts < 1 { + attempts = 1 + } + + var lastErr error + for attempt := 0; attempt < attempts; attempt++ { + if attempt > 0 && c.retryWait > 0 { + select { + case <-time.After(c.retryWait): + case <-req.Context().Done(): + return nil, req.Context().Err() + } + } + + if err := c.wait(req.Context()); err != nil { + return nil, err + } + + currentReq := req + if attempt > 0 { + currentReq = req.Clone(req.Context()) + } + + resp, err := c.client.Do(currentReq) + if err != nil { + lastErr = err + continue + } + + if resp.StatusCode >= 500 { + lastErr = fmt.Errorf("server error: %d", resp.StatusCode) + resp.Body.Close() + continue + } + + return resp, nil + } + + return nil, lastErr +} + +// GET performs a GET request applying the configured policies. func (c *ComicClient) Get(link, hostname string) (*http.Response, error) { request, err := c.PrepareRequest(link, hostname) - if err != nil { return nil, err } - return c.Client.Do(request) + return c.Do(request) +} + +func (c *ComicClient) wait(ctx context.Context) error { + if c.rateLimiter == nil { + return nil + } + return c.rateLimiter.Wait(ctx) } diff --git a/pkg/http/client_test.go b/pkg/http/client_test.go index 2572c63e..e06117d7 100644 --- a/pkg/http/client_test.go +++ b/pkg/http/client_test.go @@ -1,26 +1,90 @@ package http import ( + "context" + "net/http" + "net/http/httptest" + "sync/atomic" "testing" + "time" - "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) +type stubLimiter struct { + count int32 +} + +func (s *stubLimiter) Wait(ctx context.Context) error { + atomic.AddInt32(&s.count, 1) + return nil +} + func TestPrepareRequestMangakakalot(t *testing.T) { cc := NewComicClient() link := "http://mangakakalot.com" source := "mangakakalot.com" req, err := cc.PrepareRequest(link, source) - assert.Equal(t, req.Header["Referer"], []string{link}) - assert.Nil(t, err) + require.NoError(t, err) + require.Equal(t, link, req.Header.Get("Referer")) + require.Equal(t, defaultUserAgent, req.Header.Get("User-Agent")) } -func TestPrepareRequest(t *testing.T) { +func TestPrepareRequestGenericHost(t *testing.T) { cc := NewComicClient() link := "http://foo.com" req, err := cc.PrepareRequest(link, "foo") - assert.Equal(t, len(req.Header["Referer"]), 0) - assert.Nil(t, err) + require.NoError(t, err) + require.Empty(t, req.Header.Values("Referer")) + require.Equal(t, defaultUserAgent, req.Header.Get("User-Agent")) +} + +func TestGetRetriesOnServerError(t *testing.T) { + var hits int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + call := atomic.AddInt32(&hits, 1) + if call == 1 { + w.WriteHeader(http.StatusInternalServerError) + return + } + w.WriteHeader(http.StatusOK) + })) + defer server.Close() + + client := NewComicClient( + WithHTTPClient(server.Client()), + WithRetry(1, 0), + ) + + resp, err := client.Get(server.URL, "example.com") + require.NoError(t, err) + require.NotNil(t, resp) + require.Equal(t, int32(2), atomic.LoadInt32(&hits)) +} + +func TestRateLimiterInvoked(t *testing.T) { + limiter := &stubLimiter{} + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + })) + defer server.Close() + + client := NewComicClient( + WithHTTPClient(server.Client()), + WithRetry(0, 0), + WithRateLimiter(limiter), + ) + + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + + req, err := client.PrepareRequest(server.URL, "example.com") + require.NoError(t, err) + + req = req.WithContext(ctx) + _, err = client.Do(req) + require.NoError(t, err) + require.Equal(t, int32(1), atomic.LoadInt32(&limiter.count)) } From f927e4a67748eeaf2ce9ba9b75e977b49780700c Mon Sep 17 00:00:00 2001 From: hollisticated-horse Date: Mon, 20 Oct 2025 23:58:55 +0200 Subject: [PATCH 04/79] Refactor MangaDex scraper to use new HTTP helpers --- pkg/sites/http_helpers.go | 81 ++++++++++++ pkg/sites/mangadex.go | 162 ++++++++++++++--------- pkg/sites/mangadex_test.go | 264 +++++++++++++++++-------------------- 3 files changed, 305 insertions(+), 202 deletions(-) create mode 100644 pkg/sites/http_helpers.go diff --git a/pkg/sites/http_helpers.go b/pkg/sites/http_helpers.go new file mode 100644 index 00000000..5347b527 --- /dev/null +++ b/pkg/sites/http_helpers.go @@ -0,0 +1,81 @@ +package sites + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + urlpkg "net/url" + + httpclient "github.com/Girbons/comics-downloader/pkg/http" +) + +func defaultClient(client *httpclient.ComicClient) *httpclient.ComicClient { + if client != nil { + return client + } + return httpclient.NewComicClient() +} + +func hostFromURL(link string) string { + parsed, err := urlpkg.Parse(link) + if err != nil { + return "" + } + return parsed.Host +} + +func buildRequest(ctx context.Context, client *httpclient.ComicClient, link string) (*http.Request, error) { + req, err := client.PrepareRequest(link, hostFromURL(link)) + if err != nil { + return nil, err + } + if ctx != nil { + req = req.WithContext(ctx) + } + return req, nil +} + +func fetchJSON(ctx context.Context, client *httpclient.ComicClient, link string, target interface{}) error { + if target == nil { + return fmt.Errorf("target cannot be nil") + } + + data, err := fetchBytes(ctx, client, link) + if err != nil { + return err + } + if len(data) == 0 { + return fmt.Errorf("empty response for %s", link) + } + return json.Unmarshal(data, target) +} + +func fetchBytes(ctx context.Context, client *httpclient.ComicClient, link string) ([]byte, error) { + cc := defaultClient(client) + if ctx == nil { + ctx = context.Background() + } + + req, err := buildRequest(ctx, cc, link) + if err != nil { + return nil, err + } + + resp, err := cc.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices { + return nil, fmt.Errorf("unexpected status code %d for %s", resp.StatusCode, link) + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + return body, nil +} diff --git a/pkg/sites/mangadex.go b/pkg/sites/mangadex.go index a550c812..e7e3b942 100644 --- a/pkg/sites/mangadex.go +++ b/pkg/sites/mangadex.go @@ -1,39 +1,68 @@ package sites import ( + "context" "encoding/json" "errors" "fmt" - "net/http" + "net/url" "strings" + "time" "github.com/Girbons/comics-downloader/pkg/config" "github.com/Girbons/comics-downloader/pkg/core" + httpclient "github.com/Girbons/comics-downloader/pkg/http" "github.com/Girbons/comics-downloader/pkg/util" ) +const ( + mangadexAPIBase = "https://api.mangadex.org" + mangadexChapterBase = "https://mangadex.org/chapter" + mangadexUploadsBase = "https://uploads.mangadex.org/data" + mangadexRequestTimeout = 8 * time.Second +) + // Mangadex represents a mangadex instance. type Mangadex struct { - country string - baseURL string - options *config.Options + country string + options *config.Options + client *httpclient.ComicClient + apiBase string + chapterBase string + uploadsBase string } -// NewMangadex returns a Mangadex instance +// NewMangadex returns a Mangadex instance. func NewMangadex(options *config.Options) *Mangadex { + client := options.Client + if client == nil { + client = httpclient.NewComicClient() + options.Client = client + } + return &Mangadex{ - country: strings.ToLower(options.Country), - options: options, + country: strings.ToLower(options.Country), + options: options, + client: client, + apiBase: mangadexAPIBase, + chapterBase: mangadexChapterBase, + uploadsBase: mangadexUploadsBase, } } -func (m *Mangadex) getManga(mangaID string) (title string, err error) { - url := fmt.Sprintf("https://api.mangadex.org/manga/%s", mangaID) - res, err := http.Get(url) - if err != nil { - return "", err - } - defer res.Body.Close() +func (m *Mangadex) requestContext() (context.Context, context.CancelFunc) { + return context.WithTimeout(context.Background(), mangadexRequestTimeout) +} + +func joinURL(base, suffix string) string { + return strings.TrimRight(base, "/") + "/" + strings.TrimLeft(suffix, "/") +} + +func (m *Mangadex) getManga(mangaID string) (string, error) { + ctx, cancel := m.requestContext() + defer cancel() + + endpoint := joinURL(m.apiBase, fmt.Sprintf("/manga/%s", mangaID)) var mangaRes struct { Result string `json:"result"` Data struct { @@ -42,77 +71,87 @@ func (m *Mangadex) getManga(mangaID string) (title string, err error) { } `json:"attributes"` } `json:"data"` } - if err := json.NewDecoder(res.Body).Decode(&mangaRes); err != nil { + + if err := fetchJSON(ctx, m.client, endpoint, &mangaRes); err != nil { return "", err } - if mangaRes.Result != "ok" { - return "", fmt.Errorf("Unexpected response") + if strings.ToLower(mangaRes.Result) != "ok" { + return "", fmt.Errorf("unexpected response") } + for lang, t := range mangaRes.Data.Attributes.Titles { - title = t - if m.country == "" || m.country == lang { - break + if m.country == "" || m.country == strings.ToLower(lang) { + return t, nil } } - return title, nil + + // Fallback to any available title. + for _, t := range mangaRes.Data.Attributes.Titles { + return t, nil + } + + return "", fmt.Errorf("no title found for manga %s", mangaID) } -// Get a list of chapter IDs of a manga. +// getChapters fetches chapter URLs for the given manga. func (m *Mangadex) getChapters(mangaID string) ([]string, error) { - url := fmt.Sprintf("https://api.mangadex.org/manga/%s/aggregate?", mangaID) + ctx, cancel := m.requestContext() + defer cancel() + + endpoint := joinURL(m.apiBase, fmt.Sprintf("/manga/%s/aggregate", mangaID)) if m.country != "" { - url += fmt.Sprintf("&translatedLanguage[]=%s", m.country) + q := url.Values{} + q.Add("translatedLanguage[]", m.country) + endpoint += "?" + q.Encode() } - res, err := http.Get(url) + + body, err := fetchBytes(ctx, m.client, endpoint) if err != nil { return nil, err } - defer res.Body.Close() + if len(body) == 0 || body[0] == '[' { + return []string{}, nil + } + var chaptersRes struct { Result string `json:"result"` Volumes map[string]struct { - Name string `json:"volume"` Chapters map[string]struct { ID string `json:"id"` Name string `json:"chapter"` } `json:"chapters"` } `json:"volumes"` } - if err := json.NewDecoder(res.Body).Decode(&chaptersRes); err != nil { - // This is not ideal. MangaDex returns an empty array (`[]`) when no volume - // exists and a `map[string]interface{}` otherwise. - return []string{}, nil + + if err := json.Unmarshal(body, &chaptersRes); err != nil { + return nil, err } - if chaptersRes.Result != "ok" { - return nil, fmt.Errorf("Unexpected response") + if strings.ToLower(chaptersRes.Result) != "ok" { + return nil, fmt.Errorf("unexpected response") } + var ids []string for _, v := range chaptersRes.Volumes { for _, c := range v.Chapters { - url := fmt.Sprintf("https://mangadex.org/chapter/%s", c.ID) - ids = append(ids, url) + ids = append(ids, joinURL(m.chapterBase, c.ID)) } } return ids, nil } -// Get a list of a chapters images. +// getChapter retrieves metadata and image links for a single chapter. func (m *Mangadex) getChapter(chapterID string) (mangaID, volume, chapter, title string, images []string, err error) { - url := fmt.Sprintf("https://api.mangadex.org/chapter/%s", chapterID) - res, err := http.Get(url) - if err != nil { - return "", "", "", "", nil, err - } - defer res.Body.Close() + ctx, cancel := m.requestContext() + defer cancel() + + endpoint := joinURL(m.apiBase, fmt.Sprintf("/chapter/%s", chapterID)) var chapterRes struct { Result string `json:"result"` Data struct { Attributes struct { - Volume string `json:"volume"` - Chapter string `json:"chapter"` - Title string `json:"title"` - Hash string `json:"hash"` - Data []string `json:"data"` + Volume string `json:"volume"` + Chapter string `json:"chapter"` + Title string `json:"title"` } `json:"attributes"` Relationships []struct { ID string `json:"id"` @@ -121,13 +160,14 @@ func (m *Mangadex) getChapter(chapterID string) (mangaID, volume, chapter, title } `json:"data"` } - if err := json.NewDecoder(res.Body).Decode(&chapterRes); err != nil { + if err := fetchJSON(ctx, m.client, endpoint, &chapterRes); err != nil { return "", "", "", "", nil, err } - if chapterRes.Result != "ok" { - return "", "", "", "", nil, fmt.Errorf("Unexpected response") + if strings.ToLower(chapterRes.Result) != "ok" { + return "", "", "", "", nil, fmt.Errorf("unexpected response") } + imagesEndpoint := joinURL(m.apiBase, fmt.Sprintf("/at-home/server/%s", chapterID)) var imagesRes struct { Result string `json:"result"` Chapter struct { @@ -136,20 +176,22 @@ func (m *Mangadex) getChapter(chapterID string) (mangaID, volume, chapter, title } `json:"chapter"` } - res, err = http.Get(fmt.Sprintf("https://api.mangadex.org/at-home/server/%s", chapterID)) - - if err := json.NewDecoder(res.Body).Decode(&imagesRes); err != nil { + if err := fetchJSON(ctx, m.client, imagesEndpoint, &imagesRes); err != nil { return "", "", "", "", nil, err } + if strings.ToLower(imagesRes.Result) != "ok" { + return "", "", "", "", nil, fmt.Errorf("unexpected response") + } for _, file := range imagesRes.Chapter.Data { - imageUrl := fmt.Sprintf("https://uploads.mangadex.org/data/%s/%s", imagesRes.Chapter.Hash, file) - images = append(images, imageUrl) + imageURL := joinURL(m.uploadsBase, fmt.Sprintf("%s/%s", imagesRes.Chapter.Hash, file)) + images = append(images, imageURL) } - if m.options.Debug { + if m.options.Debug && len(images) > 0 && m.options.Logger != nil { m.options.Logger.Debug(fmt.Sprintf("Image Links found: %s", strings.Join(images, " "))) } + for _, rel := range chapterRes.Data.Relationships { if rel.Type == "manga" { mangaID = rel.ID @@ -177,8 +219,8 @@ func (m *Mangadex) RetrieveIssueLinks() ([]string, error) { } // GetInfo extracts the basic info from the given url. -func (m *Mangadex) GetInfo(url string) (string, string) { - parts := util.TrimAndSplitURL(url) +func (m *Mangadex) GetInfo(urlValue string) (string, string) { + parts := util.TrimAndSplitURL(urlValue) if len(parts) < 5 { return "", "" } @@ -208,10 +250,10 @@ func (m *Mangadex) GetInfo(url string) (string, string) { } } -// Initialize loads links and metadata from mangadex +// Initialize loads links and metadata from mangadex. func (m *Mangadex) Initialize(comic *core.Comic) error { parts := util.TrimAndSplitURL(comic.URLSource) - if len(parts) < 4 { + if len(parts) < 5 { return fmt.Errorf("URL not supported") } _, _, _, _, images, err := m.getChapter(parts[4]) diff --git a/pkg/sites/mangadex_test.go b/pkg/sites/mangadex_test.go index 6c9167ea..92353413 100644 --- a/pkg/sites/mangadex_test.go +++ b/pkg/sites/mangadex_test.go @@ -1,144 +1,124 @@ package sites -//import ( -//"testing" - -//"github.com/Girbons/comics-downloader/internal/logger" -//"github.com/Girbons/comics-downloader/pkg/config" -//"github.com/Girbons/comics-downloader/pkg/core" -//"github.com/stretchr/testify/assert" -//) - -//const testMangadexBase string = "mangadex.org" -//const testMangadexURL string = "https://" + testMangadexBase + "/" - -//func TestMangadexGetInfo(t *testing.T) { -//opt := &config.Options{ -//Url: testMangadexURL + "chapter/155061/1", -//Country: "", -//Source: testMangadexBase, -//Debug: false, -//Logger: logger.NewLogger(false, make(chan string)), -//} -//md := NewMangadex(opt) -//name, issueNumber := md.GetInfo(testMangadexURL + "chapter/155061/1") - -//assert.Equal(t, "Naruto", name) -//assert.Equal(t, "Vol 60 Chapter 575, A Will of Stone", issueNumber) -//} - -//func TestMangadexSetup(t *testing.T) { -//opt := &config.Options{ -//Url: testMangadexURL + "chapter/155061/1", -//Country: "", -//Source: testMangadexBase, -//Debug: false, -//Logger: logger.NewLogger(false, make(chan string)), -//} -//comic := new(core.Comic) -//comic.URLSource = testMangadexURL + "chapter/155061/1" - -//md := NewMangadex(opt) -//err := md.Initialize(comic) - -//assert.Nil(t, err) -//assert.Equal(t, 14, len(comic.Links)) -//} - -//func TestMangadexRetrieveIssueLinks(t *testing.T) { -//opt := &config.Options{ -//Url: testMangadexURL + "chapter/155061/", -//Country: "", -//Source: testMangadexBase, -//Last: false, -//All: false, -//Debug: false, -//Logger: logger.NewLogger(false, make(chan string)), -//} -//md := NewMangadex(opt) -//urls, err := md.RetrieveIssueLinks() -//assert.Nil(t, err) -//assert.Equal(t, 1, len(urls)) -//} - -//func TestMangadexRetrieveIssueLinksAllChapter(t *testing.T) { -//opt := &config.Options{ -//Url: testMangadexURL + "title/5/naruto/", -//Country: "gb", -//Source: testMangadexBase, -//Last: false, -//All: true, -//Debug: false, -//Logger: logger.NewLogger(false, make(chan string)), -//} -//md := NewMangadex(opt) -//urls, err := md.RetrieveIssueLinks() -//assert.Nil(t, err) -//assert.Len(t, urls, 713) -//} - -//func TestMangadexRetrieveIssueLinksLastChapter(t *testing.T) { -//opt := &config.Options{ -//Url: testMangadexURL + "title/5/naruto/", -//Country: "gb", -//Source: testMangadexBase, -//Last: true, -//All: false, -//Debug: false, -//Logger: logger.NewLogger(false, make(chan string)), -//} -//md := NewMangadex(opt) -//urls, err := md.RetrieveIssueLinks() -//assert.Nil(t, err) -//assert.Len(t, urls, 1) -//} - -//func TestMangadexUnsupportedURL(t *testing.T) { -//opt := &config.Options{ -//Url: testMangadexURL, -//Country: "", -//Source: testMangadexBase, -//Last: false, -//All: false, -//Debug: false, -//Logger: logger.NewLogger(false, make(chan string)), -//} -//md := NewMangadex(opt) -//_, err := md.RetrieveIssueLinks() -//assert.EqualError(t, err, "URL not supported") - -//md.options.Url = testMangadexURL + "test/0/" -//_, err = md.RetrieveIssueLinks() -//assert.EqualError(t, err, "URL not supported") -//} - -//func TestMangadexNoManga(t *testing.T) { -//opt := &config.Options{ -//Url: testMangadexURL + "title/0/", -//Country: "", -//Source: testMangadexBase, -//Last: false, -//All: false, -//Debug: false, -//Logger: logger.NewLogger(false, make(chan string)), -//} -//md := NewMangadex(opt) -//_, err := md.RetrieveIssueLinks() -//assert.Error(t, err) -//assert.Contains(t, err.Error(), "could not get manga 0") -//} - -//func TestMangadexNoChapters(t *testing.T) { -//opt := &config.Options{ -//Url: testMangadexURL + "title/5/naruto/", -//Country: "xyz", -//Source: testMangadexBase, -//Last: false, -//All: true, -//Debug: false, -//Logger: logger.NewLogger(false, make(chan string)), -//} -//md := NewMangadex(opt) -//_, err := md.RetrieveIssueLinks() -//assert.EqualError(t, err, "no chapters found") -//} +import ( + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/Girbons/comics-downloader/internal/logger" + "github.com/Girbons/comics-downloader/pkg/config" + "github.com/Girbons/comics-downloader/pkg/core" + httpclient "github.com/Girbons/comics-downloader/pkg/http" + "github.com/stretchr/testify/require" +) + +func setupMangadexServer() *httptest.Server { + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case strings.HasPrefix(r.URL.Path, "/manga/series-1/aggregate"): + w.Header().Set("Content-Type", "application/json") + fmt.Fprint(w, `{ + "result":"ok", + "volumes":{ + "1":{ + "chapters":{ + "1":{"id":"chapter-1","chapter":"1"} + } + } + } + }`) + case strings.HasPrefix(r.URL.Path, "/manga/series-1"): + w.Header().Set("Content-Type", "application/json") + fmt.Fprint(w, `{ + "result":"ok", + "data":{ + "attributes":{ + "title":{"en":"Test Manga","jp":"テスト"} + } + } + }`) + case strings.HasPrefix(r.URL.Path, "/chapter/chapter-1"): + w.Header().Set("Content-Type", "application/json") + fmt.Fprint(w, `{ + "result":"ok", + "data":{ + "attributes":{"volume":"1","chapter":"1","title":"Start"}, + "relationships":[{"id":"series-1","type":"manga"}] + } + }`) + case strings.HasPrefix(r.URL.Path, "/at-home/server/chapter-1"): + w.Header().Set("Content-Type", "application/json") + fmt.Fprint(w, `{ + "result":"ok", + "chapter":{"hash":"HASH","data":["001.png","002.png"]} + }`) + default: + http.NotFound(w, r) + } + })) +} + +func newTestMangadex(t *testing.T) (*Mangadex, func()) { + t.Helper() + + server := setupMangadexServer() + + client := httpclient.NewComicClient( + httpclient.WithHTTPClient(server.Client()), + httpclient.WithRetry(0, 0), + ) + + opts := &config.Options{ + URL: server.URL + "/title/series-1/naruto", + Country: "en", + Source: "mangadex.org", + Logger: logger.NewLogger(false, nil), + Client: client, + } + + md := NewMangadex(opts) + md.apiBase = server.URL + md.chapterBase = server.URL + "/chapter" + md.uploadsBase = server.URL + "/data" + + cleanup := func() { + server.Close() + } + + return md, cleanup +} + +func TestMangadexRetrieveIssueLinks(t *testing.T) { + md, cleanup := newTestMangadex(t) + defer cleanup() + + md.options.All = true + + links, err := md.RetrieveIssueLinks() + require.NoError(t, err) + require.Equal(t, []string{md.chapterBase + "/chapter-1"}, links) +} + +func TestMangadexInitialize(t *testing.T) { + md, cleanup := newTestMangadex(t) + defer cleanup() + + comic := &core.Comic{URLSource: md.chapterBase + "/chapter-1"} + err := md.Initialize(comic) + require.NoError(t, err) + require.Equal(t, []string{ + md.uploadsBase + "/HASH/001.png", + md.uploadsBase + "/HASH/002.png", + }, comic.Links) +} + +func TestMangadexGetInfo(t *testing.T) { + md, cleanup := newTestMangadex(t) + defer cleanup() + + title, chapter := md.GetInfo(md.chapterBase + "/chapter-1") + require.Equal(t, "Test Manga", title) + require.Equal(t, "Vol 1 Chapter 1, Start", chapter) +} From 18d721c1d0bba8951d117126be4a958725a14ff6 Mon Sep 17 00:00:00 2001 From: hollisticated-horse Date: Tue, 21 Oct 2025 00:13:32 +0200 Subject: [PATCH 05/79] Refactor core image pipeline and tighten tests --- pkg/core/core.go | 330 +++++++++++++++++++++--------------------- pkg/core/core_test.go | 250 +++++++++++++++----------------- 2 files changed, 278 insertions(+), 302 deletions(-) diff --git a/pkg/core/core.go b/pkg/core/core.go index 468aef26..5e69112c 100644 --- a/pkg/core/core.go +++ b/pkg/core/core.go @@ -7,14 +7,18 @@ import ( "image" "os" "path" + "path/filepath" "runtime" "sort" "strings" + "sync" + "time" "golang.org/x/sync/errgroup" "golang.org/x/sync/semaphore" "github.com/Girbons/comics-downloader/pkg/config" + httpclient "github.com/Girbons/comics-downloader/pkg/http" "github.com/Girbons/comics-downloader/pkg/util" epub "github.com/bmaupin/go-epub" "github.com/jung-kurt/gofpdf" @@ -45,63 +49,46 @@ type Comic struct { ImagesFormat string } -// makeEPUB create the epub file -func (comic *Comic) makeEPUB(options *config.Options) error { - var err error +// DownloadResult captures the outcome of downloading a comic's images. +type DownloadResult struct { + Dir string + FilePaths []string +} - currentDir, err := util.CurrentDir() - if err != nil { - return err +func ensureClient(options *config.Options) *httpclient.ComicClient { + if options.Client == nil { + options.Client = httpclient.NewComicClient() } - // used to check if the epub cover already exists + return options.Client +} + +// makeEPUB creates the epub file. +func (comic *Comic) makeEPUB(options *config.Options, images *DownloadResult) error { isCoverSet := false - // used to add the image in the epub section imgTag := `Cover Image` - // setup a new Epub instance e := epub.NewEpub(comic.IssueNumber) - // set Epub title e.SetTitle(fmt.Sprintf("%s-%s", comic.Name, comic.IssueNumber)) - // check if the author exists for this comic + if comic.Author != "" { e.SetAuthor(comic.Author) } - imagesPath, err := comic.DownloadImages(options) - if err != nil { - return err - } - defer os.RemoveAll(imagesPath) - - files, err := os.ReadDir(imagesPath) - if err != nil { - return err - } - - for _, file := range files { - // add the image to the epub will return a path - imgpath, err := e.AddImage(fmt.Sprintf("%s/%s", imagesPath, file.Name()), "") - if err != nil { + for _, file := range images.FilePaths { + imgpath, err := e.AddImage(file, "") + if err != nil && options.Logger != nil { options.Logger.Error(err.Error()) + continue } - // if the cover is not set use the first image - // otherwise the image will be added as a section if !isCoverSet { isCoverSet = true e.SetCover(imgpath, "") - } else { - _, err = e.AddSection(fmt.Sprintf(imgTag, imgpath), "", "", "") - if err != nil { - options.Logger.Error(err.Error()) - } + continue + } + if _, err := e.AddSection(fmt.Sprintf(imgTag, imgpath), "", "", ""); err != nil && options.Logger != nil { + options.Logger.Error(err.Error()) } } - if err = os.Chdir(currentDir); err != nil { - return err - } - - // get the PathSetup where the file should be saved - // e.g. /www.mangarock.com/comic-name/ dir, err := util.PathSetup(options.CreateDefaultPath, options.OutputFolder, comic.Source, comic.Name) if err != nil { return err @@ -111,50 +98,41 @@ func (comic *Comic) makeEPUB(options *config.Options) error { return err } - options.Logger.Info(fmt.Sprintf("%s %s", strings.ToUpper(comic.Format), DefaultMessage)) - return err + if options.Logger != nil { + options.Logger.Info(fmt.Sprintf("%s %s", strings.ToUpper(comic.Format), DefaultMessage)) + } + return nil } -// makePDF create the pdf file -func (comic *Comic) makePDF(options *config.Options) error { - var err error +// makePDF create the pdf file. +func (comic *Comic) makePDF(options *config.Options, images *DownloadResult) error { var mmWd, mmHt float64 const px2mm = 0.2645833333 pdf := gofpdf.New("P", "mm", "A4", "") - imagesPath, err := comic.DownloadImages(options) - if err != nil { - return err - } - - defer os.RemoveAll(imagesPath) - - files, err := os.ReadDir(imagesPath) - if err != nil { - return err - } - imageOptions := gofpdf.ImageOptions{ImageType: util.ImageType(comic.ImagesFormat), ReadDpi: true, AllowNegativePosition: false} - for _, file := range files { + for _, fileName := range images.FilePaths { mmWd = 210.0 mmHt = 297.0 - fileName := fmt.Sprintf("%s/%s", imagesPath, file.Name()) if !options.ForceAspect { img, err := os.Open(fileName) if err != nil { - options.Logger.Error(err.Error()) - } - - defer img.Close() - - im, _, err := image.DecodeConfig(img) - if err != nil { - options.Logger.Error(err.Error()) + if options.Logger != nil { + options.Logger.Error(err.Error()) + } } else { - mmWd = px2mm * float64(im.Width) - mmHt = px2mm * float64(im.Height) + im, _, err := image.DecodeConfig(img) + img.Close() + if err != nil { + if options.Logger != nil { + options.Logger.Error(err.Error()) + } + } else { + mmWd = px2mm * float64(im.Width) + mmHt = px2mm * float64(im.Height) + } } } pdf.AddPageFormat("P", gofpdf.SizeType{Wd: mmWd, Ht: mmHt}) @@ -164,8 +142,8 @@ func (comic *Comic) makePDF(options *config.Options) error { return err } content := bytes.NewReader(data) - pdf.RegisterImageOptionsReader(file.Name(), imageOptions, content) - pdf.ImageOptions(file.Name(), 0, 0, mmWd, mmHt, false, imageOptions, 0, "") + pdf.RegisterImageOptionsReader(path.Base(fileName), imageOptions, content) + pdf.ImageOptions(path.Base(fileName), 0, 0, mmWd, mmHt, false, imageOptions, 0, "") } dir, err := util.PathSetup(options.CreateDefaultPath, options.OutputFolder, comic.Source, comic.Name) @@ -173,76 +151,45 @@ func (comic *Comic) makePDF(options *config.Options) error { return err } - // Save the pdf file filePath := util.GetPathToFile(dir, comic.Name, comic.IssueNumber, comic.Format, options.IssueNumberNameOnly) if err = pdf.OutputFileAndClose(filePath); err != nil { return err } - options.Logger.Info(fmt.Sprintf("%s %s", strings.ToUpper(comic.Format), DefaultMessage)) - return err -} - -// makeCBRZ will create the CBR/CBZ -func (comic *Comic) makeCBRZ(options *config.Options) error { - var filesToAdd []string - var err error - - imagesPath, err := comic.DownloadImages(options) - if err != nil { - return err - } - defer os.RemoveAll(imagesPath) - - files, err := os.ReadDir(imagesPath) - if err != nil { - return err - } - - for _, file := range files { - filesToAdd = append(filesToAdd, fmt.Sprintf("%s/%s", imagesPath, file.Name())) + if options.Logger != nil { + options.Logger.Info(fmt.Sprintf("%s %s", strings.ToUpper(comic.Format), DefaultMessage)) } + return nil +} - // e.g. /www.mangarock.com/comic-name/ +// makeCBRZ will create the CBR/CBZ. +func (comic *Comic) makeCBRZ(options *config.Options, images *DownloadResult) error { dir, err := util.PathSetup(options.CreateDefaultPath, options.OutputFolder, comic.Source, comic.Name) if err != nil { return err } - // the archive must be created as `.zip` then change the extension to `.cbr` or `.cbz`. - zipArchiveName := fmt.Sprintf("%s/%s.zip", dir, comic.IssueNumber) + zipArchiveName := filepath.Join(dir, fmt.Sprintf("%s.zip", comic.IssueNumber)) newName := util.GetPathToFile(dir, comic.Name, comic.IssueNumber, comic.Format, options.IssueNumberNameOnly) - // Create output file out, err := os.Create(zipArchiveName) if err != nil { return err } defer out.Close() - // Sort files to ensure consistent ordering - sort.Slice(files, func(i, j int) bool { - return files[i].Name() < files[j].Name() - }) - // Map files on disk to their paths in the archive + fileMap := make(map[string]string) - for _, filePath := range filesToAdd { - // Use just the filename (no path) in the archive - fileName := path.Base(filePath) - fileMap[filePath] = fileName + for _, filePath := range images.FilePaths { + fileMap[filePath] = path.Base(filePath) } - // Get files from disk using the archives helper archiveFiles, err := archives.FilesFromDisk(context.Background(), nil, fileMap) if err != nil { return err } - // Create ZIP format (no compression needed for CBZ) format := archives.Zip{} - - // Create the archive - err = format.Archive(context.Background(), out, archiveFiles) - if err != nil { + if err = format.Archive(context.Background(), out, archiveFiles); err != nil { return err } @@ -250,116 +197,161 @@ func (comic *Comic) makeCBRZ(options *config.Options) error { return err } - options.Logger.Info(fmt.Sprintf("%s %s", strings.ToUpper(comic.Format), DefaultMessage)) + if options.Logger != nil { + options.Logger.Info(fmt.Sprintf("%s %s", strings.ToUpper(comic.Format), DefaultMessage)) + } return nil } -// DownloadImages will download the comic/manga images -func (comic *Comic) DownloadImages(options *config.Options) (string, error) { +// DownloadImages will download the comic/manga images. +func (comic *Comic) DownloadImages(options *config.Options) (*DownloadResult, error) { if len(comic.Links) == 0 { - return "", fmt.Errorf("Download failed, no links found for: %s", comic.URLSource) + return nil, fmt.Errorf("download failed, no links found for: %s", comic.URLSource) } - var dir string - var err error + client := ensureClient(options) - dir, err = util.ImagesPathSetup(options.CreateDefaultPath, options.OutputFolder, comic.Source, comic.Name, options.IssueFolderName, comic.IssueNumber) + dir, err := util.ImagesPathSetup(options.CreateDefaultPath, options.OutputFolder, comic.Source, comic.Name, options.IssueFolderName, comic.IssueNumber) if err != nil { - return dir, err + return nil, err } - files, err := os.ReadDir(dir) - if err != nil { - return dir, err + existing, err := readExistingImages(dir) + if err == nil && len(existing) == len(comic.Links) && len(existing) > 0 { + return &DownloadResult{Dir: dir, FilePaths: existing}, nil } - if !util.DirectoryOrFileDoesNotExist(dir) && len(files) == len(comic.Links) { - return dir, err + if err := os.RemoveAll(dir); err != nil { + return nil, err + } + if err := os.MkdirAll(dir, os.ModePerm); err != nil { + return nil, err } + progress := progressbar.NewOptions(len(comic.Links), progressbar.OptionSetRenderBlankState(true)) format := util.ImageType(comic.ImagesFormat) - currentDir, err := util.CurrentDir() - if err != nil { - return dir, err + type downloadJob struct { + index int + link string } - // setup the progress bar - bar := progressbar.NewOptions(len(comic.Links), progressbar.OptionSetRenderBlankState(true)) - err = os.Chdir(dir) - if err != nil { - return dir, err + jobs := make([]downloadJob, 0, len(comic.Links)) + for idx, link := range comic.Links { + if strings.TrimSpace(link) == "" { + continue + } + jobs = append(jobs, downloadJob{index: idx, link: link}) } - g := new(errgroup.Group) - - maxWorkers := int64(runtime.NumCPU()) - sem := semaphore.NewWeighted(maxWorkers) - ctx := context.Background() + results := make([]string, len(comic.Links)) + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() - defer sem.Acquire(ctx, maxWorkers) - for i, link := range comic.Links { - link, i := link, i - sem.Acquire(ctx, 1) + group, ctx := errgroup.WithContext(ctx) + sem := semaphore.NewWeighted(int64(runtime.NumCPU())) + var mu sync.Mutex - if link == "" { - continue + for _, job := range jobs { + job := job + if err := sem.Acquire(ctx, 1); err != nil { + return nil, err } - - g.Go(func() error { + group.Go(func() error { defer sem.Release(1) - rsp, err := options.Client.Get(link, comic.Source) + + reqCtx, cancelReq := context.WithTimeout(ctx, 30*time.Second) + defer cancelReq() + + request, err := client.PrepareRequest(job.link, comic.Source) if err != nil { return err } - defer rsp.Body.Close() + request = request.WithContext(reqCtx) - imgName := fmt.Sprintf("%04d-image.%s", i, format) - imgFile, err := os.Create(imgName) + response, err := client.Do(request) if err != nil { return err } - defer imgFile.Close() + defer response.Body.Close() - isWebp := strings.HasSuffix(link, ".webp") - err = util.SaveImage(imgFile, rsp.Body, format, isWebp) + fileName := fmt.Sprintf("%04d-image.%s", job.index, format) + targetPath := filepath.Join(dir, fileName) + imgFile, err := os.Create(targetPath) if err != nil { - msgError := fmt.Sprintf("There was an error while downloading image number: %d - comic issue: %s", i, comic.IssueNumber) - options.Logger.Error(msgError) - os.Remove(imgName) + return err } - if barErr := bar.Add(1); barErr != nil { - options.Logger.Error(barErr.Error()) + isWebp := strings.HasSuffix(strings.ToLower(job.link), ".webp") + if err := util.SaveImage(imgFile, response.Body, format, isWebp); err != nil { + if options.Logger != nil { + options.Logger.Error(fmt.Sprintf("There was an error while downloading image number: %d - comic issue: %s (%v)", job.index, comic.IssueNumber, err)) + } + imgFile.Close() + _ = os.Remove(targetPath) + } else { + imgFile.Close() + mu.Lock() + results[job.index] = targetPath + mu.Unlock() + } + + if progressErr := progress.Add(1); progressErr != nil && options.Logger != nil { + options.Logger.Error(progressErr.Error()) } return nil }) } - if err := g.Wait(); err != nil { - return dir, err + if err := group.Wait(); err != nil { + return nil, err } - err = os.Chdir(currentDir) + paths := filterEmpty(results) + sort.Strings(paths) + return &DownloadResult{Dir: dir, FilePaths: paths}, nil +} + +func readExistingImages(dir string) ([]string, error) { + entries, err := os.ReadDir(dir) if err != nil { - return dir, err + return nil, err } + var files []string + for _, entry := range entries { + if entry.IsDir() { + continue + } + files = append(files, filepath.Join(dir, entry.Name())) + } + sort.Strings(files) + return files, nil +} - return dir, err +func filterEmpty(items []string) []string { + var filtered []string + for _, item := range items { + if item != "" { + filtered = append(filtered, item) + } + } + return filtered } // MakeComic will create the file based on the output format selected. func (comic *Comic) MakeComic(options *config.Options) error { - var err error + result, err := comic.DownloadImages(options) + if err != nil { + return err + } + defer os.RemoveAll(result.Dir) switch comic.Format { case EPUB: - err = comic.makeEPUB(options) + return comic.makeEPUB(options, result) case CBR, CBZ: - err = comic.makeCBRZ(options) + return comic.makeCBRZ(options, result) default: - err = comic.makePDF(options) + return comic.makePDF(options, result) } - - return err } diff --git a/pkg/core/core_test.go b/pkg/core/core_test.go index e87e73dc..889341ca 100644 --- a/pkg/core/core_test.go +++ b/pkg/core/core_test.go @@ -1,177 +1,161 @@ package core import ( - "fmt" + "bytes" + "encoding/base64" + "image" + _ "image/png" + "io" + "net/http" + "net/http/httptest" "os" "path/filepath" "testing" - "time" "github.com/Girbons/comics-downloader/internal/logger" "github.com/Girbons/comics-downloader/pkg/config" - "github.com/Girbons/comics-downloader/pkg/http" - "github.com/stretchr/testify/assert" + httpclient "github.com/Girbons/comics-downloader/pkg/http" + "github.com/stretchr/testify/require" ) -func exists(f string) bool { - _, err := os.Stat(f) - if os.IsNotExist(err) { - return false - } - return err == nil -} - -func TestNewComic(t *testing.T) { - comic := new(Comic) - // links - links := []string{"foo.example.com"} - - comic.Name = "foo" - comic.IssueNumber = "2" - comic.Links = links - comic.Source = "bar" - comic.ImagesFormat = "png" - - assert.Equal(t, "foo", comic.Name) - assert.Equal(t, "2", comic.IssueNumber) - assert.Equal(t, "bar", comic.Source) - - assert.Equal(t, 1, len(comic.Links)) +var samplePNG = func() []byte { + data, _ := base64.StdEncoding.DecodeString( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADElEQVR4nGP4//8/AAX+Av4N70a4AAAAAElFTkSuQmCC", + ) + return data +}() + +func newImageServer() *httptest.Server { + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "image/png") + _, _ = w.Write(samplePNG) + })) } -func TestMakeComicPDF(t *testing.T) { - comic := new(Comic) +func newTestOptions(t *testing.T, server *httptest.Server) *config.Options { + t.Helper() - comic.Name = "foo" - comic.Format = "pdf" - comic.IssueNumber = "example-chapter-1" - comic.Links = []string{"https://via.placeholder.com/150", "https://via.placeholder.com/150", "https://via.placeholder.com/150"} - comic.ImagesFormat = "png" + client := httpclient.NewComicClient( + httpclient.WithHTTPClient(server.Client()), + httpclient.WithRetry(0, 0), + ) - opt := &config.Options{ - OutputFolder: filepath.Dir(os.Args[0]), + return &config.Options{ + OutputFolder: t.TempDir(), CreateDefaultPath: true, Debug: false, - Logger: logger.NewLogger(false, make(chan string)), - Client: http.NewComicClient(), + Logger: logger.NewLogger(false, nil), + Client: client, + IssueFolderName: "issue-", } - time.Sleep(5 * time.Second) - err := comic.MakeComic(opt) - assert.Nil(t, err) - - dir, _ := filepath.Abs(fmt.Sprintf("%s/%s/%s/%s/", filepath.Dir(os.Args[0]), "comics", "foo", "foo-example-chapter-1.pdf")) - assert.True(t, exists(dir)) } -func TestMakeComicEPUB(t *testing.T) { - comic := new(Comic) +func buildLinks(server *httptest.Server, count int) []string { + links := make([]string, count) + for i := 0; i < count; i++ { + links[i] = server.URL + "/img.png" + } + return links +} - comic.Name = "foo" - comic.Format = "epub" - comic.IssueNumber = "example-chapter-1" - comic.Author = "author" - comic.ImagesFormat = "png" +func TestDownloadImagesCreatesFiles(t *testing.T) { + server := newImageServer() + defer server.Close() - comic.Links = []string{"https://via.placeholder.com/150", "https://via.placeholder.com/150", "https://via.placeholder.com/150"} + opts := newTestOptions(t, server) - opt := &config.Options{ - OutputFolder: filepath.Dir(os.Args[0]), - CreateDefaultPath: true, - Debug: false, - Logger: logger.NewLogger(false, make(chan string)), - Client: http.NewComicClient(), + comic := &Comic{ + Name: "foo", + Source: "test-source", + IssueNumber: "1", + ImagesFormat: "png", + Links: buildLinks(server, 3), } - time.Sleep(10 * time.Second) - err := comic.MakeComic(opt) - assert.Nil(t, err) - - dir, _ := filepath.Abs(fmt.Sprintf("%s/%s/%s/%s/", filepath.Dir(os.Args[0]), "comics", "foo", "foo-example-chapter-1.epub")) - assert.True(t, exists(dir)) + result, err := comic.DownloadImages(opts) + require.NoError(t, err) + require.Len(t, result.FilePaths, 3) + + for _, file := range result.FilePaths { + data, err := readFile(file) + require.NoError(t, err) + require.NotEmpty(t, data) + _, format, err := image.Decode(bytes.NewReader(data)) + require.NoError(t, err) + require.Equal(t, "png", format) + } } -func TestDownloadImagesPNGFormat(t *testing.T) { - comic := new(Comic) - - comic.Name = "foo-png" - comic.Source = "fake" - comic.IssueNumber = "example-chapter-1" - comic.Links = []string{"https://via.placeholder.com/150", "https://via.placeholder.com/150", "https://via.placeholder.com/150"} - comic.ImagesFormat = "png" - - opt := &config.Options{ - OutputFolder: filepath.Dir(os.Args[0]), - Debug: false, - CreateDefaultPath: true, - Logger: logger.NewLogger(false, make(chan string)), - Client: http.NewComicClient(), +func TestMakeComicPDF(t *testing.T) { + server := newImageServer() + defer server.Close() + + opts := newTestOptions(t, server) + + comic := &Comic{ + Name: "foo", + Source: "test-source", + IssueNumber: "1", + Format: PDF, + ImagesFormat: "png", + Links: buildLinks(server, 2), } - time.Sleep(10 * time.Second) - _, err := comic.DownloadImages(opt) - assert.Nil(t, err) -} -func TestDownloadImagesJPGFormat(t *testing.T) { - comic := new(Comic) + require.NoError(t, comic.MakeComic(opts)) - comic.Name = "foo-jpg" - comic.Source = "fake" - comic.IssueNumber = "example-chapter-1" - comic.Links = []string{"https://via.placeholder.com/150", "https://via.placeholder.com/150", "https://via.placeholder.com/150"} - comic.ImagesFormat = "jpg" + output := filepath.Join(opts.OutputFolder, "comics", comic.Source, comic.Name, "foo-1.pdf") + require.FileExists(t, output) +} - opt := &config.Options{ - OutputFolder: filepath.Dir(os.Args[0]), - CreateDefaultPath: true, - Debug: false, - Logger: logger.NewLogger(false, make(chan string)), - Client: http.NewComicClient(), +func TestMakeComicEPUB(t *testing.T) { + server := newImageServer() + defer server.Close() + + opts := newTestOptions(t, server) + + comic := &Comic{ + Name: "bar", + Source: "test-source", + IssueNumber: "42", + Author: "Author", + Format: EPUB, + ImagesFormat: "png", + Links: buildLinks(server, 2), } - time.Sleep(10 * time.Second) - _, err := comic.DownloadImages(opt) - assert.Nil(t, err) + require.NoError(t, comic.MakeComic(opts)) + + output := filepath.Join(opts.OutputFolder, "comics", comic.Source, comic.Name, "bar-42.epub") + require.FileExists(t, output) } -func TestDownloadImagesJPEGFormat(t *testing.T) { - comic := new(Comic) +func TestMakeComicCBZ(t *testing.T) { + server := newImageServer() + defer server.Close() - comic.Name = "bar-jpeg" - comic.Source = "fake" - comic.IssueNumber = "example-chapter-1" - comic.ImagesFormat = "jpeg" - comic.Links = []string{"https://via.placeholder.com/150", "https://via.placeholder.com/150", "https://via.placeholder.com/150"} + opts := newTestOptions(t, server) - opt := &config.Options{ - OutputFolder: filepath.Dir(os.Args[0]), - CreateDefaultPath: true, - Debug: false, - Logger: logger.NewLogger(false, make(chan string)), - Client: http.NewComicClient(), + comic := &Comic{ + Name: "baz", + Source: "test-source", + IssueNumber: "7", + Format: CBZ, + ImagesFormat: "png", + Links: buildLinks(server, 2), } - time.Sleep(10 * time.Second) - _, err := comic.DownloadImages(opt) - assert.Nil(t, err) -} -func TestDownloadImagesIMGFormat(t *testing.T) { - comic := new(Comic) + require.NoError(t, comic.MakeComic(opts)) - comic.Name = "bar-img" - comic.Source = "fake" - comic.IssueNumber = "example-chapter-1" - comic.Links = []string{"https://via.placeholder.com/150", "https://via.placeholder.com/150", "https://via.placeholder.com/150"} - comic.ImagesFormat = "img" + output := filepath.Join(opts.OutputFolder, "comics", comic.Source, comic.Name, "baz-7.cbz") + require.FileExists(t, output) +} - opt := &config.Options{ - OutputFolder: filepath.Dir(os.Args[0]), - CreateDefaultPath: true, - Debug: false, - Logger: logger.NewLogger(false, make(chan string)), - Client: http.NewComicClient(), +func readFile(path string) ([]byte, error) { + file, err := os.Open(path) + if err != nil { + return nil, err } - time.Sleep(10 * time.Second) - _, err := comic.DownloadImages(opt) + defer file.Close() - assert.Nil(t, err) + return io.ReadAll(file) } From d1a50f8f4e86b73da9f27a811e655f37678d9796 Mon Sep 17 00:00:00 2001 From: hollisticated-horse Date: Tue, 21 Oct 2025 00:18:11 +0200 Subject: [PATCH 06/79] Improve logger safety and add formatted helpers --- cmd/app/downloader.go | 4 +- internal/logger/customlogger.go | 89 ++++++++++++++++++++-------- internal/logger/customlogger_test.go | 46 ++++++++++++++ pkg/core/core.go | 8 +-- 4 files changed, 116 insertions(+), 31 deletions(-) create mode 100644 internal/logger/customlogger_test.go diff --git a/cmd/app/downloader.go b/cmd/app/downloader.go index a4fd3a85..7f6b56db 100644 --- a/cmd/app/downloader.go +++ b/cmd/app/downloader.go @@ -114,7 +114,7 @@ func (r *Runner) download(base config.Options) { if outputFolder == "" { dir, err := os.Getwd() if err != nil { - opts.Logger.Error(fmt.Sprintf("Error determining current directory: %v", err)) + opts.Logger.Errorf("Error determining current directory: %v", err) outputFolder = "." } else { outputFolder = dir @@ -130,7 +130,7 @@ func (r *Runner) download(base config.Options) { } if isNewVersionAvailable { - opts.Logger.Info(fmt.Sprintf("A new comics-downloader version is available at %s", newVersionLink)) + opts.Logger.Infof("A new comics-downloader version is available at %s", newVersionLink) } for _, rawURL := range strings.Split(opts.URL, ",") { diff --git a/internal/logger/customlogger.go b/internal/logger/customlogger.go index be278a4c..0e5cfb7e 100644 --- a/internal/logger/customlogger.go +++ b/internal/logger/customlogger.go @@ -2,57 +2,96 @@ package logger import ( "fmt" + "strings" + "sync" "github.com/sirupsen/logrus" ) -// Logger is the custom app logger. +// Logger is the custom app logger that can optionally emit messages to a GUI channel. type Logger struct { - l *logrus.Logger + inner *logrus.Logger bindToChannel bool - Messages chan string + messages chan string + channelMu sync.RWMutex } -// NewLogger returns a logger instance +// NewLogger returns a logger instance. func NewLogger(bindToChannel bool, messages chan string) *Logger { + log := logrus.New() + log.SetFormatter(&logrus.TextFormatter{DisableTimestamp: true}) return &Logger{ - l: logrus.New(), - Messages: messages, - bindToChannel: bindToChannel, + inner: log, + bindToChannel: bindToChannel && messages != nil, + messages: messages, } } -// SetLevel set logger level. +// SetLevel sets the logger level. func (logger *Logger) SetLevel(level logrus.Level) { - logger.l.SetLevel(level) + logger.inner.SetLevel(level) } -func (logger *Logger) sendToChannel(msg string) { - if logger.bindToChannel { - logger.Messages <- msg +// Writer exposes the underlying logrus logger for advanced usage. +func (logger *Logger) Writer() *logrus.Logger { + return logger.inner +} + +func (logger *Logger) sendToChannel(level, msg string) { + logger.channelMu.RLock() + defer logger.channelMu.RUnlock() + + if !logger.bindToChannel || logger.messages == nil { + return + } + + formatted := fmt.Sprintf("%s: %s", strings.ToUpper(level), msg) + select { + case logger.messages <- formatted: + default: } } -// Info logs info level log. +// Debug logs at Debug level. +func (logger *Logger) Debug(msg string) { + logger.inner.Debug(msg) + logger.sendToChannel("DEBUG", msg) +} + +// Debugf logs formatted entries at Debug level. +func (logger *Logger) Debugf(format string, args ...interface{}) { + logger.Debug(fmt.Sprintf(format, args...)) +} + +// Info logs at Info level. func (logger *Logger) Info(msg string) { - logger.l.Info(msg) - logger.sendToChannel(fmt.Sprintf("INFO: %s", msg)) + logger.inner.Info(msg) + logger.sendToChannel("INFO", msg) } -// Debug logs debug level log. -func (logger *Logger) Debug(msg string) { - logger.l.Debug(msg) - logger.sendToChannel(fmt.Sprintf("DEBUG: %s", msg)) +// Infof logs formatted entries at Info level. +func (logger *Logger) Infof(format string, args ...interface{}) { + logger.Info(fmt.Sprintf(format, args...)) } -// Warning logs Warning level log. +// Warning logs at Warning level. func (logger *Logger) Warning(msg string) { - logger.l.Warning(msg) - logger.sendToChannel(fmt.Sprintf("WARNING: %s", msg)) + logger.inner.Warning(msg) + logger.sendToChannel("WARNING", msg) } -// Error logs error level log. +// Warningf logs formatted entries at Warning level. +func (logger *Logger) Warningf(format string, args ...interface{}) { + logger.Warning(fmt.Sprintf(format, args...)) +} + +// Error logs at Error level. func (logger *Logger) Error(msg string) { - logger.l.Error(msg) - logger.sendToChannel(fmt.Sprintf("ERROR: %s", msg)) + logger.inner.Error(msg) + logger.sendToChannel("ERROR", msg) +} + +// Errorf logs formatted entries at Error level. +func (logger *Logger) Errorf(format string, args ...interface{}) { + logger.Error(fmt.Sprintf(format, args...)) } diff --git a/internal/logger/customlogger_test.go b/internal/logger/customlogger_test.go new file mode 100644 index 00000000..c22659c8 --- /dev/null +++ b/internal/logger/customlogger_test.go @@ -0,0 +1,46 @@ +package logger + +import ( + "testing" + "time" + + "github.com/sirupsen/logrus" +) + +func TestInfofSendsToChannel(t *testing.T) { + ch := make(chan string, 1) + log := NewLogger(true, ch) + log.SetLevel(logrus.InfoLevel) + + log.Infof("downloaded %d files", 3) + + select { + case msg := <-ch: + if msg != "INFO: downloaded 3 files" { + t.Fatalf("unexpected message %q", msg) + } + default: + t.Fatalf("expected message to be published") + } +} + +func TestChannelSendDoesNotBlock(t *testing.T) { + ch := make(chan string, 1) + log := NewLogger(true, ch) + + log.Info("first") + log.Info("second") // should not block even if buffer is full + + select { + case <-ch: + case <-time.After(100 * time.Millisecond): + t.Fatalf("expected to dequeue at least one message") + } +} + +func TestLoggerHandlesNilChannel(t *testing.T) { + log := NewLogger(false, nil) + log.Debug("noop") + log.Infof("hello %s", "world") + log.Errorf("error %s", "msg") +} diff --git a/pkg/core/core.go b/pkg/core/core.go index 5e69112c..cc0e8fb4 100644 --- a/pkg/core/core.go +++ b/pkg/core/core.go @@ -99,7 +99,7 @@ func (comic *Comic) makeEPUB(options *config.Options, images *DownloadResult) er } if options.Logger != nil { - options.Logger.Info(fmt.Sprintf("%s %s", strings.ToUpper(comic.Format), DefaultMessage)) + options.Logger.Infof("%s %s", strings.ToUpper(comic.Format), DefaultMessage) } return nil } @@ -157,7 +157,7 @@ func (comic *Comic) makePDF(options *config.Options, images *DownloadResult) err } if options.Logger != nil { - options.Logger.Info(fmt.Sprintf("%s %s", strings.ToUpper(comic.Format), DefaultMessage)) + options.Logger.Infof("%s %s", strings.ToUpper(comic.Format), DefaultMessage) } return nil } @@ -198,7 +198,7 @@ func (comic *Comic) makeCBRZ(options *config.Options, images *DownloadResult) er } if options.Logger != nil { - options.Logger.Info(fmt.Sprintf("%s %s", strings.ToUpper(comic.Format), DefaultMessage)) + options.Logger.Infof("%s %s", strings.ToUpper(comic.Format), DefaultMessage) } return nil } @@ -285,7 +285,7 @@ func (comic *Comic) DownloadImages(options *config.Options) (*DownloadResult, er isWebp := strings.HasSuffix(strings.ToLower(job.link), ".webp") if err := util.SaveImage(imgFile, response.Body, format, isWebp); err != nil { if options.Logger != nil { - options.Logger.Error(fmt.Sprintf("There was an error while downloading image number: %d - comic issue: %s (%v)", job.index, comic.IssueNumber, err)) + options.Logger.Errorf("There was an error while downloading image number: %d - comic issue: %s (%v)", job.index, comic.IssueNumber, err) } imgFile.Close() _ = os.Remove(targetPath) From a5a7f30364ed9050328241f04a2eac0fe42b033e Mon Sep 17 00:00:00 2001 From: hollisticated-horse Date: Tue, 21 Oct 2025 00:23:54 +0200 Subject: [PATCH 07/79] Add golangci-lint configuration and workflow docs --- .golangci.yml | 22 ++++++++++++++++++++++ .travis.yml | 2 ++ AGENTS.md | 2 +- Makefile | 3 +++ docs/dev.md | 14 ++++++++++++++ 5 files changed, 42 insertions(+), 1 deletion(-) create mode 100644 .golangci.yml diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 00000000..2fb4bbfe --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,22 @@ +run: + timeout: 5m + tests: false + skip-dirs: + - comics + +linters: + enable: + - errcheck + - govet + - staticcheck + - gosimple + - ineffassign + - typecheck + +linters-settings: + errcheck: + check-type-assertions: true + check-blank: true + +issues: + exclude-use-default: false diff --git a/.travis.yml b/.travis.yml index 9ef54714..9bbde714 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,8 +6,10 @@ go: before_install: - sudo apt-get update - sudo apt install libgl1-mesa-dev xorg-dev + - go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.59.1 - go install github.com/mattn/goveralls@latest script: + - $(go env GOPATH)/bin/golangci-lint run - go test -v ./... - $GOPATH/bin/goveralls -service=travis-ci diff --git a/AGENTS.md b/AGENTS.md index ee856cc7..38e92590 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -4,7 +4,7 @@ The project targets Go 1.23+ with a standard layout. `cmd/downloader` hosts the CLI entrypoint, while `cmd/gui` wraps the GUI built with Fyne; both depend on reusable packages under `pkg/`. Key subpackages include `pkg/core` for orchestration, `pkg/sites` for site-specific scrapers, and `pkg/util` for image and filesystem helpers. Supporting flag parsing and other internals live in `internal/`. Docs for contributors live in `docs/`, assets in `img/`, and release artifacts appear under `build/` when generated. ## Build, Test, and Development Commands -Run `go build -o ./bin/comics-downloader ./cmd/downloader` for a local CLI binary. The GUI can be built with `go build -o ./bin/comics-downloader-gui ./cmd/gui` once Fyne prerequisites are met. Cross-platform artifacts are automated via Make targets such as `make linux-x86-64-build` or the aggregate `make builds`. Use `fyne-cross windows -output comics-downloader-gui-windows.exe ./cmd/gui` when Docker-based cross-compilation is preferred. Always execute `go test -v ./...` before pushing; CI mirrors this command and reports coverage with Coveralls. +Run `go build -o ./bin/comics-downloader ./cmd/downloader` for a local CLI binary. The GUI can be built with `go build -o ./bin/comics-downloader-gui ./cmd/gui` once Fyne prerequisites are met. Cross-platform artifacts are automated via Make targets such as `make linux-x86-64-build` or the aggregate `make builds`. Use `fyne-cross windows -output comics-downloader-gui-windows.exe ./cmd/gui` when Docker-based cross-compilation is preferred. Static checks run via `make lint` (wraps `golangci-lint run` and matches CI), and `go test -v ./...` should pass before pushing; CI mirrors this command set and reports coverage with Coveralls. ## Coding Style & Naming Conventions Format all Go files with `gofmt` (invoked via `go fmt ./...`) to preserve canonical tab-indented style and import ordering. Follow idiomatic Go naming: exported identifiers use CamelCase, unexported use camelCase, constants are ALL_CAPS only when enumerations require clarity. Keep packages small and cohesive; prefer descriptive filenames like `mangadex.go` aligning with the site handled. Log through the existing Logrus helpers to stay consistent with current output. diff --git a/Makefile b/Makefile index 2a98b479..38a37e3b 100644 --- a/Makefile +++ b/Makefile @@ -50,3 +50,6 @@ builds: # Creates executables for OSX/Windows/Linux remove-builds: # Remove executables @rm -rf build/ + +lint: # Run static analysis + @golangci-lint run diff --git a/docs/dev.md b/docs/dev.md index f903e3fe..4fbf68f3 100644 --- a/docs/dev.md +++ b/docs/dev.md @@ -34,3 +34,17 @@ which requires [Docker](https://www.docker.com/get-started). ``` go test -v ./... ``` + +## Lint + +Install golangci-lint once with: + +``` +go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.59.1 +``` + +Then run the full static analysis suite with: + +``` +golangci-lint run +``` From 4d1b798397b963b8da0efcfb6eefc965f25e03f1 Mon Sep 17 00:00:00 2001 From: hollisticated-horse Date: Tue, 21 Oct 2025 00:32:22 +0200 Subject: [PATCH 08/79] Normalize obfuscated Comicextra image URLs --- pkg/sites/comicextra.go | 6 +++--- pkg/sites/comicextra_deobfuscate_test.go | 21 +++++++++++++++++++++ pkg/sites/deobfuscate.go | 24 ++++++++++++++++++++++++ 3 files changed, 48 insertions(+), 3 deletions(-) create mode 100644 pkg/sites/comicextra_deobfuscate_test.go create mode 100644 pkg/sites/deobfuscate.go diff --git a/pkg/sites/comicextra.go b/pkg/sites/comicextra.go index 07db0ac4..a31f8228 100644 --- a/pkg/sites/comicextra.go +++ b/pkg/sites/comicextra.go @@ -36,9 +36,9 @@ func (c *Comicextra) retrieveImageLinks(comic *core.Comic) ([]string, error) { match := re.FindAllStringSubmatch(response, -1) for i := range match { - url := match[i][1] - if util.IsURLValid(url) { - links = append(links, url) + link := deobfuscateURL(match[i][1]) + if util.IsURLValid(link) { + links = append(links, link) } } diff --git a/pkg/sites/comicextra_deobfuscate_test.go b/pkg/sites/comicextra_deobfuscate_test.go new file mode 100644 index 00000000..d1a86ff7 --- /dev/null +++ b/pkg/sites/comicextra_deobfuscate_test.go @@ -0,0 +1,21 @@ +package sites + +import "testing" + +func TestDeobfuscateURL(t *testing.T) { + cases := []struct { + input string + expected string + }{ + {`https:\/\/foo.com\/img%3Fid%3D1`, "https://foo.com/img?id=1"}, + {"https://foo.com/img=2&token%3Dabc%252F123", "https://foo.com/img=2&token=abc%2F123"}, + {"", ""}, + } + + for _, tc := range cases { + actual := deobfuscateURL(tc.input) + if actual != tc.expected { + t.Fatalf("deobfuscateURL(%q) = %q, expected %q", tc.input, actual, tc.expected) + } + } +} diff --git a/pkg/sites/deobfuscate.go b/pkg/sites/deobfuscate.go new file mode 100644 index 00000000..1d3580e4 --- /dev/null +++ b/pkg/sites/deobfuscate.go @@ -0,0 +1,24 @@ +package sites + +import ( + "html" + "net/url" + "strings" +) + +func deobfuscateURL(raw string) string { + if raw == "" { + return "" + } + + cleaned := strings.ReplaceAll(raw, `\/`, "/") + cleaned = strings.ReplaceAll(cleaned, `\\`, `\`) + + decoded := html.UnescapeString(cleaned) + + if result, err := url.PathUnescape(decoded); err == nil { + decoded = result + } + + return decoded +} From 68047dc8c8e3c2b71242414f7018f8617fbcbbca Mon Sep 17 00:00:00 2001 From: hollisticated-horse Date: Tue, 21 Oct 2025 00:50:24 +0200 Subject: [PATCH 09/79] Add optional HTTP mitigation for protected sources --- AGENTS.md | 6 +++ README.md | 10 +++++ cmd/app/downloader.go | 41 +++++++++++++++++++-- cmd/downloader/main.go | 26 +++++++++++++ cmd/downloader/main_test.go | 16 ++++++++ docs/dev.md | 11 ++++++ pkg/config/options.go | 2 + pkg/http/client.go | 73 +++++++++++++++++++++++++++++++++---- pkg/http/client_test.go | 33 +++++++++++++++++ 9 files changed, 208 insertions(+), 10 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 38e92590..18deae9b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -6,6 +6,8 @@ The project targets Go 1.23+ with a standard layout. `cmd/downloader` hosts the ## Build, Test, and Development Commands Run `go build -o ./bin/comics-downloader ./cmd/downloader` for a local CLI binary. The GUI can be built with `go build -o ./bin/comics-downloader-gui ./cmd/gui` once Fyne prerequisites are met. Cross-platform artifacts are automated via Make targets such as `make linux-x86-64-build` or the aggregate `make builds`. Use `fyne-cross windows -output comics-downloader-gui-windows.exe ./cmd/gui` when Docker-based cross-compilation is preferred. Static checks run via `make lint` (wraps `golangci-lint run` and matches CI), and `go test -v ./...` should pass before pushing; CI mirrors this command set and reports coverage with Coveralls. +For sites with aggressive bot protections, you can rotate request headers using `--user-agents "UA1,UA2"` and forward cookies such as `cf_clearance` with `--session-cookie "cf_clearance=...; other=value"` when invoking the CLI. + ## Coding Style & Naming Conventions Format all Go files with `gofmt` (invoked via `go fmt ./...`) to preserve canonical tab-indented style and import ordering. Follow idiomatic Go naming: exported identifiers use CamelCase, unexported use camelCase, constants are ALL_CAPS only when enumerations require clarity. Keep packages small and cohesive; prefer descriptive filenames like `mangadex.go` aligning with the site handled. Log through the existing Logrus helpers to stay consistent with current output. @@ -14,3 +16,7 @@ Place tests alongside implementation in `*_test.go` files, using table-driven ca ## Commit & Pull Request Guidelines Commit subjects should be concise and imperative (`add`, `fix`, `update`), optionally prefixed with scopes like `fix:` as seen in history. Commit early and keep diffs focused; avoid mixing GUI and CLI changes in one commit. Pull requests must target `master`, include a brief summary of the change, reference related issues, and note how to reproduce or test the update (logs, commands, screenshots for GUI tweaks). Ensure tests pass locally before requesting review. + +## Security & Configuration Tips +- Use `--user-agents` to supply a comma-separated list of desktop/mobile agents; the downloader rotates them per request. +- When Cloudflare or other shields require clearance cookies, reuse your browser session via `--session-cookie` so authenticated requests succeed. diff --git a/README.md b/README.md index 471f634f..109cbfbc 100644 --- a/README.md +++ b/README.md @@ -199,6 +199,16 @@ Default is **jpg**. ./comics-downloader -url=[your url] -images-only -images-format=jpg ``` +### Work Around Site Protection + +Some sources employ rotating fingerprints or require a logged-in session. You can rotate multiple User-Agent strings and forward session cookies collected from your browser: + +```bash +./comics-downloader -url=[your url] \ + --user-agents="Mozilla/5.0 ...,Mozilla/5.0 (Macintosh; ...)" \ + --session-cookie="cf_clearance=...; other=value" +``` + ### Avoid Default Folder Structure The default folder structure that will be created is: diff --git a/cmd/app/downloader.go b/cmd/app/downloader.go index 7f6b56db..11af2a38 100644 --- a/cmd/app/downloader.go +++ b/cmd/app/downloader.go @@ -36,15 +36,15 @@ type Runner struct { // NewRunner returns a Runner with default factories. func NewRunner(base config.Options) *Runner { + clientOpts := buildClientOptions(base) + return &Runner{ base: base, loggerFactory: func(bind bool, messages chan string) *logger.Logger { return logger.NewLogger(bind, messages) }, clientFactory: func() *httpclient.ComicClient { - return httpclient.NewComicClient( - httpclient.WithUserAgent(fmt.Sprintf("comics-downloader/%s", version.Tag)), - ) + return httpclient.NewComicClient(clientOpts...) }, sleep: time.Sleep, } @@ -193,3 +193,38 @@ func Run(options *config.Options) { runner := NewRunner(*options) runner.Run() } + +func buildClientOptions(base config.Options) []httpclient.Option { + defaultUA := fmt.Sprintf("comics-downloader/%s", version.Tag) + agents := mergeUserAgents(defaultUA, base.UserAgents) + opts := []httpclient.Option{httpclient.WithUserAgents(agents)} + + if strings.TrimSpace(base.SessionCookie) != "" { + opts = append(opts, httpclient.WithHeaders(map[string]string{ + "Cookie": base.SessionCookie, + })) + } + + return opts +} + +func mergeUserAgents(defaultAgent string, provided []string) []string { + candidates := append([]string{defaultAgent}, provided...) + seen := make(map[string]struct{}, len(candidates)) + var result []string + for _, candidate := range candidates { + trimmed := strings.TrimSpace(candidate) + if trimmed == "" { + continue + } + if _, ok := seen[trimmed]; ok { + continue + } + seen[trimmed] = struct{}{} + result = append(result, trimmed) + } + if len(result) == 0 { + result = append(result, defaultAgent) + } + return result +} diff --git a/cmd/downloader/main.go b/cmd/downloader/main.go index e4930666..59e38402 100644 --- a/cmd/downloader/main.go +++ b/cmd/downloader/main.go @@ -4,6 +4,7 @@ import ( "flag" "fmt" "os" + "strings" "github.com/Girbons/comics-downloader/cmd/app" "github.com/Girbons/comics-downloader/internal/version" @@ -42,6 +43,9 @@ var ( issuesRange string // string to be used for each issue/chapter folder issueFolderName string + // request customization + userAgentsCSV string + sessionCookie string ) func init() { @@ -62,6 +66,8 @@ func init() { flag.StringVar(&outputFolder, "output", "", "Folder where the comics will be saved") flag.StringVar(&issuesRange, "range", "", "Range of issues to download, example 3-9") flag.StringVar(&issueFolderName, "issue-folder-name", "issue-", "Folder name where each issue/chapter will be saved, default 'issue-#'") + flag.StringVar(&userAgentsCSV, "user-agents", "", "Comma-separated list of alternative User-Agent values to rotate per request") + flag.StringVar(&sessionCookie, "session-cookie", "", "Custom Cookie header value (e.g., cf_clearance=...; other=...) for protected sources") flag.IntVar(&daemonTimeout, "daemon-timeout", 600, "DaemonTimeout (seconds), specifies how often the downloader runs") } @@ -85,6 +91,8 @@ func buildOptions() config.Options { CreateDefaultPath: createDefaultPath, IssuesRange: issuesRange, IssueFolderName: issueFolderName, + UserAgents: splitAndTrim(userAgentsCSV), + SessionCookie: strings.TrimSpace(sessionCookie), } } @@ -99,3 +107,21 @@ func main() { opts := buildOptions() app.Run(&opts) } + +func splitAndTrim(csv string) []string { + if strings.TrimSpace(csv) == "" { + return nil + } + parts := strings.Split(csv, ",") + var out []string + for _, part := range parts { + trimmed := strings.TrimSpace(part) + if trimmed != "" { + out = append(out, trimmed) + } + } + if len(out) == 0 { + return nil + } + return out +} diff --git a/cmd/downloader/main_test.go b/cmd/downloader/main_test.go index db0e04e3..2a1d50ab 100644 --- a/cmd/downloader/main_test.go +++ b/cmd/downloader/main_test.go @@ -23,6 +23,8 @@ func TestBuildOptionsCopiesGlobals(t *testing.T) { daemonTimeout int issuesRange string issueFolderName string + userAgentsCSV string + sessionCookie string }{ debug: debug, all: all, @@ -41,6 +43,8 @@ func TestBuildOptionsCopiesGlobals(t *testing.T) { daemonTimeout: daemonTimeout, issuesRange: issuesRange, issueFolderName: issueFolderName, + userAgentsCSV: userAgentsCSV, + sessionCookie: sessionCookie, } defer func() { debug = prev.debug @@ -60,6 +64,8 @@ func TestBuildOptionsCopiesGlobals(t *testing.T) { daemonTimeout = prev.daemonTimeout issuesRange = prev.issuesRange issueFolderName = prev.issueFolderName + userAgentsCSV = prev.userAgentsCSV + sessionCookie = prev.sessionCookie }() debug = true @@ -79,6 +85,8 @@ func TestBuildOptionsCopiesGlobals(t *testing.T) { daemonTimeout = 42 issuesRange = "1-5" issueFolderName = "chapter-" + userAgentsCSV = "UA1, UA2 ," + sessionCookie = "cf_clearance=abc123; other=value" opts := buildOptions() @@ -105,4 +113,12 @@ func TestBuildOptionsCopiesGlobals(t *testing.T) { if opts.IssuesRange != "1-5" { t.Fatalf("expected issues range to be copied, got %q", opts.IssuesRange) } + + if len(opts.UserAgents) != 2 || opts.UserAgents[0] != "UA1" || opts.UserAgents[1] != "UA2" { + t.Fatalf("expected user agents to be parsed, got %+v", opts.UserAgents) + } + + if opts.SessionCookie != "cf_clearance=abc123; other=value" { + t.Fatalf("expected session cookie to be copied, got %q", opts.SessionCookie) + } } diff --git a/docs/dev.md b/docs/dev.md index 4fbf68f3..37157537 100644 --- a/docs/dev.md +++ b/docs/dev.md @@ -48,3 +48,14 @@ Then run the full static analysis suite with: ``` golangci-lint run ``` + +### Cloudflare / Request Tweaks + +Some sources require browser-like fingerprints. The CLI accepts: + +``` +--user-agents "UA1,UA2" # rotate these agents per request +--session-cookie "cf_clearance=...; other=value" +``` + +Capture values from a working browser session when needed. diff --git a/pkg/config/options.go b/pkg/config/options.go index b03d5a70..cdfb092f 100644 --- a/pkg/config/options.go +++ b/pkg/config/options.go @@ -25,6 +25,8 @@ type Options struct { Source string IssuesRange string IssueFolderName string + UserAgents []string + SessionCookie string Client *http.ComicClient Logger *logger.Logger diff --git a/pkg/http/client.go b/pkg/http/client.go index f2d02c42..2b84218e 100644 --- a/pkg/http/client.go +++ b/pkg/http/client.go @@ -6,6 +6,7 @@ import ( "fmt" "net/http" "strings" + "sync/atomic" "time" ) @@ -52,11 +53,35 @@ func WithRateLimiter(limiter RateLimiter) Option { } } -// WithUserAgent overrides the default user-agent header. +// WithUserAgent overrides the default user-agent header with a single value. func WithUserAgent(agent string) Option { + return WithUserAgents([]string{agent}) +} + +// WithUserAgents configures a list of User-Agent values to rotate through per request. +func WithUserAgents(agents []string) Option { return func(cc *ComicClient) { - if strings.TrimSpace(agent) != "" { - cc.userAgent = agent + filtered := filterNonEmpty(agents) + if len(filtered) > 0 { + cc.userAgents = filtered + } + } +} + +// WithHeaders adds static headers to every request. +func WithHeaders(headers map[string]string) Option { + return func(cc *ComicClient) { + if len(headers) == 0 { + return + } + if cc.headers == nil { + cc.headers = make(map[string]string) + } + for key, value := range headers { + if strings.TrimSpace(key) == "" || strings.TrimSpace(value) == "" { + continue + } + cc.headers[key] = value } } } @@ -67,7 +92,9 @@ type ComicClient struct { retryCount int retryWait time.Duration rateLimiter RateLimiter - userAgent string + userAgents []string + headers map[string]string + uaCounter uint32 } // NewComicClient returns a ComicClient instance with sane defaults. @@ -78,7 +105,8 @@ func NewComicClient(options ...Option) *ComicClient { }, retryCount: defaultRetryCount, retryWait: defaultRetryWait, - userAgent: defaultUserAgent, + userAgents: []string{defaultUserAgent}, + headers: make(map[string]string), } for _, opt := range options { @@ -89,6 +117,10 @@ func NewComicClient(options ...Option) *ComicClient { cc.client.Timeout = defaultTimeout } + if len(cc.userAgents) == 0 { + cc.userAgents = []string{defaultUserAgent} + } + return cc } @@ -111,8 +143,14 @@ func (c *ComicClient) PrepareRequest(link, hostname string) (*http.Request, erro req.Header.Set("Referer", link) } - if c != nil && c.userAgent != "" { - req.Header.Set("User-Agent", c.userAgent) + if c != nil { + req.Header.Set("User-Agent", c.selectUserAgent()) + for key, value := range c.headers { + if strings.EqualFold(key, "user-agent") { + continue + } + req.Header.Set(key, value) + } } return req, nil @@ -185,3 +223,24 @@ func (c *ComicClient) wait(ctx context.Context) error { } return c.rateLimiter.Wait(ctx) } + +func (c *ComicClient) selectUserAgent() string { + if len(c.userAgents) == 0 { + return defaultUserAgent + } + if len(c.userAgents) == 1 { + return c.userAgents[0] + } + index := int(atomic.AddUint32(&c.uaCounter, 1)-1) % len(c.userAgents) + return c.userAgents[index] +} + +func filterNonEmpty(values []string) []string { + var out []string + for _, value := range values { + if trimmed := strings.TrimSpace(value); trimmed != "" { + out = append(out, trimmed) + } + } + return out +} diff --git a/pkg/http/client_test.go b/pkg/http/client_test.go index e06117d7..067fd784 100644 --- a/pkg/http/client_test.go +++ b/pkg/http/client_test.go @@ -88,3 +88,36 @@ func TestRateLimiterInvoked(t *testing.T) { require.NoError(t, err) require.Equal(t, int32(1), atomic.LoadInt32(&limiter.count)) } + +func TestUserAgentRotation(t *testing.T) { + agents := []string{"UA-1", "UA-2"} + // we need to ensure rotation occurs across calls + client := NewComicClient(WithUserAgents(agents)) + + req1, err := client.PrepareRequest("http://example.com", "example.com") + require.NoError(t, err) + req2, err := client.PrepareRequest("http://example.com", "example.com") + require.NoError(t, err) + + if req1.Header.Get("User-Agent") == req2.Header.Get("User-Agent") { + t.Fatalf("expected rotating user agents, got identical headers %q", req1.Header.Get("User-Agent")) + } + + req3, err := client.PrepareRequest("http://example.com", "example.com") + require.NoError(t, err) + + require.Equal(t, req1.Header.Get("User-Agent"), req3.Header.Get("User-Agent"), "rotation should loop back to first entry") +} + +func TestAdditionalHeadersApplied(t *testing.T) { + client := NewComicClient(WithHeaders(map[string]string{ + "Cookie": "cf_clearance=abc", + "X-Custom-Id": "123", + })) + + req, err := client.PrepareRequest("http://example.com", "example.com") + require.NoError(t, err) + + require.Equal(t, "cf_clearance=abc", req.Header.Get("Cookie")) + require.Equal(t, "123", req.Header.Get("X-Custom-Id")) +} From 40c84e2f8875bc0bc4e56b44a5a720860b3ce0be Mon Sep 17 00:00:00 2001 From: hollisticated-horse Date: Tue, 21 Oct 2025 01:01:48 +0200 Subject: [PATCH 10/79] Replace MangaKakalot network tests with fixtures --- pkg/sites/mangakakalot_test.go | 120 +++++++++++++++++++++------------ 1 file changed, 78 insertions(+), 42 deletions(-) diff --git a/pkg/sites/mangakakalot_test.go b/pkg/sites/mangakakalot_test.go index 8e91110b..351709ee 100644 --- a/pkg/sites/mangakakalot_test.go +++ b/pkg/sites/mangakakalot_test.go @@ -1,59 +1,95 @@ package sites import ( + "fmt" + "net/http" + "net/http/httptest" + "strings" "testing" - "github.com/Girbons/comics-downloader/pkg/core" - "github.com/Girbons/comics-downloader/internal/logger" "github.com/Girbons/comics-downloader/pkg/config" - "github.com/stretchr/testify/assert" + "github.com/Girbons/comics-downloader/pkg/core" + "github.com/stretchr/testify/require" ) -func TestMangaKakalotGetInfo(t *testing.T) { - opt := &config.Options{ - URL: "https://mangakakalot.com/read-wa8ap158524529412", - All: false, - Last: false, - Debug: false, - Logger: logger.NewLogger(false, make(chan string)), - } - mk := NewMangaKakalot(opt) - // mangakakalot.com - name, issueNumber := mk.GetInfo("https://mangakakalot.com/chapter/evergreen/chapter_2") - assert.Equal(t, "A Case", name) - assert.Equal(t, "2", issueNumber) -} +const ( + mangaKakalotChapterPath = "/chapter/manga-title/chapter-2" + mangaKakalotListPath = "/manga/manga-title" +) -func TestMangaKakalotSetup(t *testing.T) { - opt := &config.Options{ - URL: "https://mangakakalot.com/read-wa8ap158524529412", - All: false, - Last: false, - Debug: false, - Logger: logger.NewLogger(false, make(chan string)), - } - mk := NewMangaKakalot(opt) - comic := new(core.Comic) - comic.URLSource = "https://mangakakalot.com/chapter/evergreen/chapter_2" +func newMangaKakalotServer() *httptest.Server { + chapterHTML := ` + + +
+

+ Home + Chapter 2 : My Manga +

+
+
+ + +
+ + ` - err := mk.Initialize(comic) + listHTMLTemplate := ` + + +
+
+ +
+
+ +
+
+ + ` - assert.Nil(t, err) - assert.Equal(t, 40, len(comic.Links)) + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.URL.Path == mangaKakalotChapterPath: + _, _ = fmt.Fprint(w, chapterHTML) + case r.URL.Path == mangaKakalotListPath: + base := "http://" + r.Host + _, _ = fmt.Fprintf(w, listHTMLTemplate, base, base) + case strings.HasPrefix(r.URL.Path, "/chapter/manga-title/chapter-1"): + _, _ = fmt.Fprint(w, "") + default: + http.NotFound(w, r) + } + })) } -func TestMangaKakalotRetrieveIssueLinks(t *testing.T) { - opt := &config.Options{ - URL: "https://mangakakalot.com/read-wa8ap158524529412", - All: false, - Last: false, - Debug: false, - Logger: logger.NewLogger(false, make(chan string)), +func TestMangaKakalotScraper(t *testing.T) { + server := newMangaKakalotServer() + defer server.Close() + + opts := &config.Options{ + URL: server.URL + mangaKakalotListPath, + Source: "mangakakalot.com", + Logger: logger.NewLogger(false, nil), } - mk := NewMangaKakalot(opt) - links, err := mk.RetrieveIssueLinks() + scraper := NewMangaKakalot(opts) + + title, issue := scraper.GetInfo(server.URL + mangaKakalotChapterPath) + require.Equal(t, "My Manga", title) + require.Equal(t, "2", issue) + + comic := &core.Comic{URLSource: server.URL + mangaKakalotChapterPath} + require.NoError(t, scraper.Initialize(comic)) + require.Equal(t, []string{ + "https://cdn.example.com/manga-title/001.jpg", + "https://cdn.example.com/manga-title/002.jpg", + }, comic.Links) - assert.Nil(t, err) - assert.Equal(t, 46, len(links)) + links, err := scraper.RetrieveIssueLinks() + require.NoError(t, err) + require.Equal(t, []string{ + server.URL + mangaKakalotChapterPath, + server.URL + "/chapter/manga-title/chapter-1", + }, links) } From d9eea23725eae4ae7f34fdf68a1216149cac27b4 Mon Sep 17 00:00:00 2001 From: hollisticated-horse Date: Tue, 21 Oct 2025 02:18:42 +0200 Subject: [PATCH 11/79] Replace site tests with fixture-backed servers --- pkg/sites/comicextra_test.go | 190 +++++++++++------------- pkg/sites/loader_test.go | 218 +++++++-------------------- pkg/sites/manganato_test.go | 118 +++++++++------ pkg/sites/mangareader_test.go | 192 ++++++++++++------------ pkg/sites/mangatown_test.go | 142 +++++++++++------- pkg/sites/readallcomics_test.go | 237 +++++++++++------------------- pkg/sites/readcomiconline_test.go | 137 ++++++++--------- 7 files changed, 542 insertions(+), 692 deletions(-) diff --git a/pkg/sites/comicextra_test.go b/pkg/sites/comicextra_test.go index 9256a244..16feda9a 100644 --- a/pkg/sites/comicextra_test.go +++ b/pkg/sites/comicextra_test.go @@ -1,131 +1,113 @@ package sites import ( + "net/http" + "net/http/httptest" "testing" "github.com/Girbons/comics-downloader/internal/logger" "github.com/Girbons/comics-downloader/pkg/config" "github.com/Girbons/comics-downloader/pkg/core" - "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) -func TestComicExtraSetup(t *testing.T) { - comic := new(core.Comic) - comic.URLSource = "https://comicextra.me/batman-unseen/issue-5/full" +const ( + comicExtraIssueFullPath = "/batman-unseen/issue-5/full" + comicExtraIssueSimple = "/batman-unseen/issue-5" + comicExtraLastIssuePath = "/batman-unseen/issue-4/full" + comicExtraListPath = "/comic/batman-unseen" +) - opt := - &config.Options{ - URL: "https://comicextra.me/batman-unseen/issue-5/full", - All: false, - Last: false, - Debug: false, - Logger: logger.NewLogger(false, make(chan string)), +func newComicExtraServer() *httptest.Server { + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + base := "http://" + r.Host + switch r.URL.Path { + case comicExtraIssueFullPath, comicExtraIssueSimple, "/batman-unseen/issue-4": + html := ` + + + + + + ` + _, _ = w.Write([]byte(html)) + case comicExtraLastIssuePath: + html := ` + + + + + ` + _, _ = w.Write([]byte(html)) + case comicExtraListPath: + html := ` + + +
+ + +
+ + ` + _, _ = w.Write([]byte(html)) + default: + http.NotFound(w, r) } - - comicextra := NewComicextra(opt) - err := comicextra.Initialize(comic) - - assert.Nil(t, err) - assert.Equal(t, 23, len(comic.Links)) + })) } -func TestComicExtraGetInfo(t *testing.T) { - opt := - &config.Options{ - URL: "https://comicextra.me/batman-unseen/issue-5/full", - All: false, - Last: false, - Debug: false, - Logger: logger.NewLogger(false, make(chan string)), - } +func TestComicExtraScraper(t *testing.T) { + server := newComicExtraServer() + defer server.Close() - comicextra := NewComicextra(opt) - name, issueNumber := comicextra.GetInfo("https://comicextra.me/batman-unseen/issue-5/full") + opts := &config.Options{ + URL: server.URL + comicExtraIssueFullPath, + Logger: logger.NewLogger(false, nil), + } - assert.Equal(t, "batman-unseen", name) - assert.Equal(t, "issue-5", issueNumber) -} - -func TestComicextraRetrieveIssueLinks(t *testing.T) { - opt := - &config.Options{ - URL: "https://comicextra.me/batman-unseen/issue-5/full", - All: false, - Last: false, - Debug: false, - Logger: logger.NewLogger(false, make(chan string)), - } + comicextra := NewComicextra(opts) - comicextra := NewComicextra(opt) - issues, err := comicextra.RetrieveIssueLinks() - - assert.Nil(t, err) - assert.Equal(t, 1, len(issues)) + comic := &core.Comic{URLSource: server.URL + comicExtraIssueFullPath} + require.NoError(t, comicextra.Initialize(comic)) + require.Equal(t, []string{ + "https://cdn.example.com/batman?page=1", + "https://cdn.example.com/batman?page=2", + }, comic.Links) } -func TestComicextraRetrieveIssueLinksURLWithPage(t *testing.T) { - opt := - &config.Options{ - URL: "https://comicextra.me/batman-unseen/full", - All: false, - Last: false, - Debug: false, - Logger: logger.NewLogger(false, make(chan string)), - } - - comicextra := NewComicextra(opt) - issues, err := comicextra.RetrieveIssueLinks() +func TestComicExtraRetrieveIssueLinksAll(t *testing.T) { + server := newComicExtraServer() + defer server.Close() - assert.Nil(t, err) - assert.Equal(t, 1, len(issues)) -} + opts := &config.Options{ + URL: server.URL + comicExtraListPath, + All: true, + Logger: logger.NewLogger(false, nil), + } -func TestComicextraRetrieveIssueLinksInASinglePage(t *testing.T) { - opt := - &config.Options{ - URL: "https://comicextra.me/batman-unseen/issue-4/full", - All: false, - Last: false, - Debug: false, - Logger: logger.NewLogger(false, make(chan string)), - } - - comicextra := NewComicextra(opt) - issues, err := comicextra.RetrieveIssueLinks() - - assert.Nil(t, err) - assert.Equal(t, 1, len(issues)) -} - -func TestComicextraRetrieveIssueLinksLastChapter(t *testing.T) { - opt := - &config.Options{ - URL: "https://comicextra.me/batman-unseen/issue-4/full", - All: false, - Last: true, - Debug: false, - Logger: logger.NewLogger(false, make(chan string)), - } - - comicextra := NewComicextra(opt) + comicextra := NewComicextra(opts) issues, err := comicextra.RetrieveIssueLinks() - - assert.Nil(t, err) - assert.Equal(t, 1, len(issues)) + require.NoError(t, err) + require.Equal(t, []string{ + comicExtraIssueFullPath, + "/batman-unseen/issue-4/full", + }, issues) } -func TestComicExtraRetrieveLastIssueLink(t *testing.T) { - comicextra := new(Comicextra) - issue, err := comicextra.retrieveLastIssue("https://comicextra.me/batman-unseen/issue-1/full") +func TestComicExtraRetrieveLastIssue(t *testing.T) { + server := newComicExtraServer() + defer server.Close() - assert.Nil(t, err) - assert.Equal(t, "https://comicextra.me/batman-unseen/issue-5/full", issue) -} - -func TestComicExtraRetrieveLastIssueLinkNotDetail(t *testing.T) { - comicextra := new(Comicextra) - issue, err := comicextra.retrieveLastIssue("https://comicextra.me/batman-unseen/issue-1/full") + opts := &config.Options{ + URL: server.URL + comicExtraLastIssuePath, + Last: true, + Logger: logger.NewLogger(false, nil), + } - assert.Nil(t, err) - assert.Equal(t, "https://comicextra.me/batman-unseen/issue-5/full", issue) + comicextra := NewComicextra(opts) + issues, err := comicextra.RetrieveIssueLinks() + require.NoError(t, err) + require.Equal(t, []string{server.URL + comicExtraIssueFullPath}, issues) } diff --git a/pkg/sites/loader_test.go b/pkg/sites/loader_test.go index 8ee3f9f3..6204740d 100644 --- a/pkg/sites/loader_test.go +++ b/pkg/sites/loader_test.go @@ -1,200 +1,84 @@ package sites import ( - "fmt" - "os" - "path/filepath" + "errors" "testing" "github.com/Girbons/comics-downloader/pkg/config" - "github.com/stretchr/testify/assert" + "github.com/Girbons/comics-downloader/pkg/core" + "github.com/stretchr/testify/require" ) -func TestSiteLoaderMangatown(t *testing.T) { - url := "https://www.mangatown.com/manga/naruto/v63/c693/" - outputFolder := filepath.Dir(os.Args[0]) - - options := &config.Options{ - All: false, - Last: false, - ImagesOnly: false, - Source: "www.mangatown.com", - URL: url, - Format: "pdf", - ImagesFormat: "png", - OutputFolder: outputFolder, - } - - collection, err := LoadComicFromSource(options) - - assert.Nil(t, err) - assert.Equal(t, len(collection), 1) - - comic := collection[0] - - assert.Equal(t, "www.mangatown.com", comic.Source) - assert.Equal(t, url, comic.URLSource) - assert.Equal(t, "naruto", comic.Name) - assert.Equal(t, "c693", comic.IssueNumber) - assert.Equal(t, 20, len(comic.Links)) +type stubSite struct { + issues []string + comics map[string]*core.Comic } -func TestCustomComicName(t *testing.T) { - url := "https://www.mangatown.com/manga/naruto/v63/c693/" - outputFolder := filepath.Dir(os.Args[0]) - - options := &config.Options{ - All: false, - Last: false, - ImagesOnly: false, - Source: "www.mangatown.com", - URL: url, - Format: "pdf", - ImagesFormat: "png", - CustomComicName: "Naruto", - OutputFolder: outputFolder, +func (s *stubSite) Initialize(comic *core.Comic) error { + if stub, ok := s.comics[comic.URLSource]; ok { + *comic = *stub + return nil } - - collection, err := LoadComicFromSource(options) - - assert.Nil(t, err) - assert.Equal(t, len(collection), 1) - - comic := collection[0] - - assert.Equal(t, "www.mangatown.com", comic.Source) - assert.Equal(t, url, comic.URLSource) - assert.Equal(t, "Naruto", comic.Name) - assert.Equal(t, "c693", comic.IssueNumber) - assert.Equal(t, 20, len(comic.Links)) + return errors.New("missing comic") } -//func TestSiteLoaderMangareader(t *testing.T) { -//url := "https://www.mangareader.net/naruto/700" -//outputFolder := filepath.Dir(os.Args[0]) - -//options := &config.Options{ -//All: false, -//Last: false, -//ImagesOnly: false, -//Source: "www.mangareader.net", -//Url: url, -//Format: "pdf", -//ImagesFormat: "png", -//OutputFolder: outputFolder, -//} - -//collection, err := LoadComicFromSource(options) - -//assert.Nil(t, err) -//assert.Equal(t, len(collection), 1) - -//comic := collection[0] - -//assert.Equal(t, "www.mangareader.net", comic.Source) -//assert.Equal(t, url, comic.URLSource) -//assert.Equal(t, "naruto", comic.Name) -//assert.Equal(t, "700", comic.IssueNumber) -//assert.Equal(t, 23, len(comic.Links)) -//} - -func TestSiteLoaderComicExtra(t *testing.T) { - url := "https://comicextra.me/batman-unseen/issue-5/full" - outputFolder := filepath.Dir(os.Args[0]) - options := &config.Options{ - All: false, - Last: false, - ImagesOnly: false, - Source: "comicextra.net", - URL: url, - Format: "pdf", - ImagesFormat: "png", - OutputFolder: outputFolder, +func (s *stubSite) GetInfo(url string) (string, string) { + if stub, ok := s.comics[url]; ok { + return stub.Name, stub.IssueNumber } - collection, err := LoadComicFromSource(options) - - assert.Nil(t, err) - assert.Equal(t, 1, len(collection)) - - comic := collection[0] - - assert.Equal(t, "comicextra.net", comic.Source) - assert.Equal(t, url, comic.URLSource) - assert.Equal(t, "batman-unseen", comic.Name) - assert.Equal(t, "issue-5", comic.IssueNumber) - assert.Equal(t, 23, len(comic.Links)) + return "", "" } -func TestLoaderUnknownSource(t *testing.T) { - url := "http://example.com" - outputFolder := filepath.Dir(os.Args[0]) +func (s *stubSite) RetrieveIssueLinks() ([]string, error) { + return s.issues, nil +} +func TestInitializeCollectionFiltersIssues(t *testing.T) { options := &config.Options{ - All: false, - Last: false, - ImagesOnly: false, - Source: "example.com", - URL: url, + Source: "test-source", Format: "pdf", ImagesFormat: "png", - OutputFolder: outputFolder, + IssuesRange: "1-2", + All: true, } - collection, err := LoadComicFromSource(options) - - if assert.NotNil(t, err) { - assert.Equal(t, fmt.Errorf("source unknown"), err) + site := &stubSite{ + issues: []string{"url-1", "url-2", "url-3"}, + comics: map[string]*core.Comic{ + "url-1": {Name: "series", IssueNumber: "issue-1", URLSource: "url-1"}, + "url-2": {Name: "series", IssueNumber: "issue-2", URLSource: "url-2"}, + "url-3": {Name: "series", IssueNumber: "issue-3", URLSource: "url-3"}, + }, } - assert.Equal(t, len(collection), 0) + + collection, err := initializeCollection(site.issues, options, site) + require.NoError(t, err) + require.Len(t, collection, 2) + require.Equal(t, "issue-1", collection[0].IssueNumber) + require.Equal(t, "issue-2", collection[1].IssueNumber) } -func TestIssuesRange(t *testing.T) { - url := "https://comicextra.net/batman-unseen/issue-5/full" - outputFolder := filepath.Dir(os.Args[0]) - options := &config.Options{ - All: true, - Last: false, - ImagesOnly: false, - Source: "comicextra.net", - URL: url, - Format: "pdf", - ImagesFormat: "png", - OutputFolder: outputFolder, - IssuesRange: "1-3", - } +func TestLoadComicFromSourceUnknown(t *testing.T) { + options := &config.Options{Source: "unknown"} collection, err := LoadComicFromSource(options) - - assert.Nil(t, err) - assert.Equal(t, len(collection), 3) - - issues := make([]string, 0, len(collection)) - for _, c := range collection { - issues = append(issues, c.IssueNumber) - } - - assert.Contains(t, issues, "issue-1") - assert.Contains(t, issues, "issue-2") - assert.Contains(t, issues, "issue-3") + require.Error(t, err) + require.Empty(t, collection) } -func TestFloatIssuesRange(t *testing.T) { - tt := []struct { - input string - start float64 - end float64 - returnValue bool +func TestNotInIssuesRange(t *testing.T) { + testCases := []struct { + issue string + start float64 + end float64 + skip bool }{ - {"1", 1, 1, false}, - {"19", 20, 21, true}, - {"20", 20, 21, false}, - {"20.5", 20, 21, false}, - {"21", 20, 21, false}, - {"22", 20, 21, true}, + {"1", 1, 2, false}, + {"3", 1, 2, true}, + {"2.5", 2, 3, false}, + {"abc", 1, 2, true}, } - for _, tc := range tt { - t.Run(tc.input, func(t *testing.T) { - assert.Equal(t, notInIssuesRange(tc.input, tc.start, tc.end), tc.returnValue) - }) + for _, tc := range testCases { + require.Equal(t, tc.skip, notInIssuesRange(tc.issue, tc.start, tc.end)) } } diff --git a/pkg/sites/manganato_test.go b/pkg/sites/manganato_test.go index a5866720..23a85a6d 100644 --- a/pkg/sites/manganato_test.go +++ b/pkg/sites/manganato_test.go @@ -1,59 +1,93 @@ package sites import ( + "fmt" + "net/http" + "net/http/httptest" "testing" - "github.com/Girbons/comics-downloader/pkg/core" - "github.com/Girbons/comics-downloader/internal/logger" "github.com/Girbons/comics-downloader/pkg/config" - "github.com/stretchr/testify/assert" + "github.com/Girbons/comics-downloader/pkg/core" + "github.com/stretchr/testify/require" ) -func TestManganatoGetInfo(t *testing.T) { - opt := &config.Options{ - URL: "https://chapmanganato.com/manga-ng952689", - All: false, - Last: false, - Debug: false, - Logger: logger.NewLogger(false, make(chan string)), - } - mg := NewManganato(opt) - // readmanganato.com - name, issueNumber := mg.GetInfo("https://chapmanganato.com/manga-ng952689/chapter-700.5") - assert.Equal(t, "Uzumaki Naruto", name) - assert.Equal(t, "700.5", issueNumber) -} +const ( + manganatoChapterPath = "/chapter/manga-title/chapter-2" + manganatoListPath = "/read/manga-title" +) -func TestManganatoSetup(t *testing.T) { - opt := &config.Options{ - URL: "https://chapmanganato.com/manga-ng952689", - All: false, - Last: false, - Debug: false, - Logger: logger.NewLogger(false, make(chan string)), - } - mk := NewManganato(opt) - comic := new(core.Comic) - comic.URLSource = "https://chapmanganato.com/manga-ng952689/chapter-700.5" +func newManganatoServer() *httptest.Server { + chapterHTML := ` + + +
+ Home + Chapter 2 : My Manga +
+
+ + +
+ + ` - err := mk.Initialize(comic) + listHTMLTemplate := ` + + +
+
  • + +
    • +
  • + +
    • +
    + + ` - assert.Nil(t, err) - assert.Equal(t, 18, len(comic.Links)) + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case manganatoChapterPath: + _, _ = fmt.Fprint(w, chapterHTML) + case manganatoListPath: + base := "http://" + r.Host + _, _ = fmt.Fprintf(w, listHTMLTemplate, base, base) + case "/chapter/manga-title/chapter-1": + _, _ = fmt.Fprint(w, "") + default: + http.NotFound(w, r) + } + })) } -func TestManganatoRetrieveIssueLinks(t *testing.T) { - opt := &config.Options{ - URL: "https://chapmanganato.com/manga-ng952689", - All: false, - Last: false, - Debug: false, - Logger: logger.NewLogger(false, make(chan string)), +func TestManganatoScraper(t *testing.T) { + server := newManganatoServer() + defer server.Close() + + opts := &config.Options{ + URL: server.URL + manganatoListPath, + Source: "manganato.com", + Logger: logger.NewLogger(false, nil), } - mk := NewManganato(opt) - links, err := mk.RetrieveIssueLinks() - assert.Nil(t, err) - assert.Equal(t, 748, len(links)) + scraper := NewManganato(opts) + + title, issue := scraper.GetInfo(server.URL + manganatoChapterPath) + require.Equal(t, "My Manga", title) + require.Equal(t, "2", issue) + + comic := &core.Comic{URLSource: server.URL + manganatoChapterPath} + require.NoError(t, scraper.Initialize(comic)) + require.Equal(t, []string{ + "https://cdn.example.com/manga-title/001.jpg", + "https://cdn.example.com/manga-title/002.jpg", + }, comic.Links) + + links, err := scraper.RetrieveIssueLinks() + require.NoError(t, err) + require.Equal(t, []string{ + server.URL + manganatoChapterPath, + server.URL + "/chapter/manga-title/chapter-1", + }, links) } diff --git a/pkg/sites/mangareader_test.go b/pkg/sites/mangareader_test.go index 8cbb6542..9b521c9c 100644 --- a/pkg/sites/mangareader_test.go +++ b/pkg/sites/mangareader_test.go @@ -1,103 +1,93 @@ package sites -//import ( -// "testing" -// -// "github.com/Girbons/comics-downloader/internal/logger" -// "github.com/Girbons/comics-downloader/pkg/config" -// "github.com/Girbons/comics-downloader/pkg/core" -// "github.com/stretchr/testify/assert" -//) -// -//const ( -// URL = "https://www.mangareader.net/naruto/1/" -// SOURCE = "www.mangareader.net" -//) -// -//func TestMangareaderGetInfo(t *testing.T) { -// mr := new(Mangareader) -// name, issueNumber := mr.GetInfo(URL) -// -// assert.Equal(t, "naruto", name) -// assert.Equal(t, "1", issueNumber) -//} -// -//func TestRetrieveMangareaderImageLinks(t *testing.T) { -// opt := -// &config.Options{ -// URL: URL, -// Source: SOURCE, -// All: false, -// Last: false, -// Debug: false, -// Logger: logger.NewLogger(false, make(chan string)), -// } -// mr := NewMangareader(opt) -// -// comic := new(core.Comic) -// comic.URLSource = URL -// comic.Name = "naruto" -// comic.IssueNumber = "1" -// comic.Source = SOURCE -// -// links, err := mr.retrieveImageLinks(comic) -// -// assert.Equal(t, 53, len(links)) -// assert.Nil(t, err) -//} -// -//func TestSetupMangareader(t *testing.T) { -// opt := -// &config.Options{ -// URL: URL, -// Source: SOURCE, -// All: false, -// Last: false, -// Debug: false, -// Logger: logger.NewLogger(false, make(chan string)), -// } -// mr := NewMangareader(opt) -// -// comic := new(core.Comic) -// comic.Name = "naruto" -// comic.IssueNumber = "1" -// comic.URLSource = URL -// comic.Source = SOURCE -// -// err := mr.Initialize(comic) -// -// assert.Nil(t, err) -// assert.Equal(t, 53, len(comic.Links)) -//} -// -//func TestMangareaderRetrieveIssueLinks(t *testing.T) { -// opt := -// &config.Options{ -// URL: "https://www.mangareader.net/naruto", -// All: false, -// Last: false, -// Debug: false, -// Logger: logger.NewLogger(false, make(chan string)), -// } -// mr := NewMangareader(opt) -// issues, err := mr.RetrieveIssueLinks() -// -// assert.Nil(t, err) -// assert.Equal(t, 700, len(issues)) -//} -// -//func TestMangareaderRetrieveLastIssueLink(t *testing.T) { -// opt := -// &config.Options{ -// URL: "https://www.mangareader.net/naruto", -// All: false, -// Last: true, -// Debug: false, -// Logger: logger.NewLogger(false, make(chan string)), -// } -// mr := NewMangareader(opt) -// issue, err := mr.retrieveLastIssue("https://www.mangareader.net/naruto") -// -// assert.Nil(t, err) -// assert.Equal(t, "https://www.mangareader.net/naruto/700", issue) -//} +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/Girbons/comics-downloader/internal/logger" + "github.com/Girbons/comics-downloader/pkg/config" + "github.com/Girbons/comics-downloader/pkg/core" + "github.com/stretchr/testify/require" +) + +const ( + mangareaderIssuePath = "/naruto/1/" + mangareaderBasePath = "/naruto" + mangareaderLastIssue = "/naruto/700" +) + +func newMangareaderServer() *httptest.Server { + issueHTML := ` + + + + + + ` + + baseHTML := ` + + + + + +
    Chapter 1
    Chapter 2
    + + + ` + + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case mangareaderIssuePath: + _, _ = w.Write([]byte(issueHTML)) + case "/naruto/2/": + _, _ = w.Write([]byte(issueHTML)) + case mangareaderBasePath: + _, _ = w.Write([]byte(baseHTML)) + case mangareaderLastIssue: + _, _ = w.Write([]byte("")) + default: + http.NotFound(w, r) + } + })) +} + +func TestMangareaderScraper(t *testing.T) { + server := newMangareaderServer() + defer server.Close() + + opts := &config.Options{ + URL: server.URL + mangareaderIssuePath, + Logger: logger.NewLogger(false, nil), + } + + scraper := NewMangareader(opts) + + comic := &core.Comic{URLSource: server.URL + mangareaderIssuePath} + require.NoError(t, scraper.Initialize(comic)) + require.Equal(t, []string{ + "https://cdn.example.com/naruto/001.jpg", + "https://cdn.example.com/naruto/002.jpg", + }, comic.Links) + + opts.All = true + opts.URL = server.URL + mangareaderIssuePath + scraper = NewMangareader(opts) + issues, err := scraper.RetrieveIssueLinks() + require.NoError(t, err) + require.Equal(t, []string{ + "https://mangareader.tv" + mangareaderIssuePath, + "https://mangareader.tv/naruto/2/", + }, issues) + + opts.Last = true + opts.All = false + opts.URL = server.URL + mangareaderBasePath + scraper = NewMangareader(opts) + lastIssues, err := scraper.RetrieveIssueLinks() + require.NoError(t, err) + require.Equal(t, []string{"https://mangareader.tv" + mangareaderLastIssue}, lastIssues) +} diff --git a/pkg/sites/mangatown_test.go b/pkg/sites/mangatown_test.go index d4f9ef14..79f4815e 100644 --- a/pkg/sites/mangatown_test.go +++ b/pkg/sites/mangatown_test.go @@ -1,77 +1,105 @@ package sites import ( + "fmt" + "net/http" + "net/http/httptest" "testing" "github.com/Girbons/comics-downloader/internal/logger" "github.com/Girbons/comics-downloader/pkg/config" "github.com/Girbons/comics-downloader/pkg/core" - "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) -func TestMangatownGetInfo(t *testing.T) { - opt := - &config.Options{ - URL: "http://www.mangatown.com/manga/naruto/v63/c684/", - All: false, - Last: false, - Debug: false, - Logger: logger.NewLogger(false, make(chan string)), - } - mt := NewMangatown(opt) - name, issueNumber := mt.GetInfo("http://www.mangatown.com/manga/naruto/v63/c684/") +const ( + mangatownIssuePath = "/manga/naruto/v63/c684/" +) - assert.Equal(t, "naruto", name) - assert.Equal(t, "c684", issueNumber) -} +func newMangatownServer() *httptest.Server { + firstPage := ` + + +
    + +
    +
    + + ` -func TestMangatownSetup(t *testing.T) { - opt := - &config.Options{ - URL: "http://www.mangatown.com/manga/naruto/v63/c684/", - All: false, - Last: false, - Debug: false, - Logger: logger.NewLogger(false, make(chan string)), - } - mt := NewMangatown(opt) - comic := new(core.Comic) - comic.URLSource = "http://www.mangatown.com/manga/naruto/v63/c684/" + secondPage := ` + + +
    + + ` - err := mt.Initialize(comic) + listHTML := ` + + +
      + + +
    + + ` - assert.Nil(t, err) - assert.Equal(t, 22, len(comic.Links)) + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case mangatownIssuePath: + _, _ = fmt.Fprint(w, firstPage) + case "/manga/naruto/v63/c684/1.html": + _, _ = fmt.Fprint(w, firstPage) + case "/manga/naruto/v63/c684/2.html": + _, _ = fmt.Fprint(w, secondPage) + case "/manga/naruto/v63": + _, _ = fmt.Fprint(w, listHTML) + case "/manga/naruto": + _, _ = fmt.Fprint(w, listHTML) + case "/manga/naruto/": + _, _ = fmt.Fprint(w, listHTML) + default: + http.NotFound(w, r) + } + })) } -func TestMangatownRetrieveIssueLinks(t *testing.T) { - opt := - &config.Options{ - URL: "http://www.mangatown.com/manga/naruto/v63/c684/", - All: true, - Last: false, - Debug: false, - Logger: logger.NewLogger(false, make(chan string)), - } - mt := NewMangatown(opt) - issues, err := mt.RetrieveIssueLinks() +func TestMangatownScraper(t *testing.T) { + server := newMangatownServer() + defer server.Close() - assert.Nil(t, err) - assert.Equal(t, 752, len(issues)) -} + opts := &config.Options{ + URL: server.URL + mangatownIssuePath, + Logger: logger.NewLogger(false, nil), + } -func TestMangatownRetrieveIssueLinksLastChapter(t *testing.T) { - opt := - &config.Options{ - URL: "http://www.mangatown.com/manga/naruto/", - All: false, - Last: true, - Debug: false, - Logger: logger.NewLogger(false, make(chan string)), - } - mt := NewMangatown(opt) - issues, err := mt.RetrieveIssueLinks() + scraper := NewMangatown(opts) + + comic := &core.Comic{URLSource: server.URL + mangatownIssuePath} + require.NoError(t, scraper.Initialize(comic)) + require.Equal(t, []string{ + "https://cdn.example.com/naruto/001.jpg", + "https://cdn.example.com/naruto/002.jpg", + }, comic.Links) + + opts.All = true + scraper = NewMangatown(opts) + issues, err := scraper.RetrieveIssueLinks() + require.NoError(t, err) + require.Equal(t, []string{ + "https://mangatown.com/manga/naruto/v63/c684/", + "https://mangatown.com/manga/naruto/v63/c685/", + }, issues) - assert.Nil(t, err) - assert.Equal(t, 1, len(issues)) + opts.All = false + opts.Last = true + opts.URL = server.URL + "/manga/naruto/" + scraper = NewMangatown(opts) + last, err := scraper.RetrieveIssueLinks() + require.NoError(t, err) + require.Equal(t, []string{"https://www.mangatown.com/manga/naruto/v63/c684/"}, last) } diff --git a/pkg/sites/readallcomics_test.go b/pkg/sites/readallcomics_test.go index feed8a1d..340eb795 100644 --- a/pkg/sites/readallcomics_test.go +++ b/pkg/sites/readallcomics_test.go @@ -1,171 +1,110 @@ package sites import ( + "net/http" + "net/http/httptest" "testing" "github.com/Girbons/comics-downloader/internal/logger" "github.com/Girbons/comics-downloader/pkg/config" "github.com/Girbons/comics-downloader/pkg/core" - "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) -func TestReadallcomicsSetup(t *testing.T) { - comic := new(core.Comic) - comic.URLSource = "https://readallcomics.com/sandman-v2-075-1989/" - opt := &config.Options{ - URL: "https://readallcomics.com/sandman-v2-075-1989/", - All: false, - Last: false, - Debug: false, - Logger: logger.NewLogger(false, make(chan string)), - } - readallcomics := NewReadallcomics(opt) - err := readallcomics.Initialize(comic) - assert.Nil(t, err) - // Note: This would depend on the actual page content, adjust expected count as needed - assert.Greater(t, len(comic.Links), 0) -} - -func TestReadallcomicsGetInfoSomethingIsKillingTheChildren(t *testing.T) { - opt := &config.Options{ - URL: "https://readallcomics.com/something-is-killing-the-children-000-2024/", - All: false, - Last: false, - Debug: false, - Logger: logger.NewLogger(false, make(chan string)), - } - readallcomics := NewReadallcomics(opt) - name, issueNumber := readallcomics.GetInfo("https://readallcomics.com/something-is-killing-the-children-000-2024/") - assert.Equal(t, "something is killing the children", name) - assert.Equal(t, "000-2024", issueNumber) -} - -func TestReadallcomicsGetInfoEmbeddedIssue(t *testing.T) { - opt := &config.Options{ - URL: "https://readallcomics.com/something-is-killing-the-children-029something-is-killing-the-children-2023/", - All: false, - Last: false, - Debug: false, - Logger: logger.NewLogger(false, make(chan string)), - } - readallcomics := NewReadallcomics(opt) - name, issueNumber := readallcomics.GetInfo("https://readallcomics.com/something-is-killing-the-children-029something-is-killing-the-children-2023/") - assert.Equal(t, "something is killing the children", name) - assert.Equal(t, "029", issueNumber) -} - -func TestReadallcomicsGetInfoSandmanV2(t *testing.T) { - opt := &config.Options{ - URL: "https://readallcomics.com/sandman-v2-075-1989/", - All: false, - Last: false, - Debug: false, - Logger: logger.NewLogger(false, make(chan string)), - } - readallcomics := NewReadallcomics(opt) - name, issueNumber := readallcomics.GetInfo("https://readallcomics.com/sandman-v2-075-1989/") - assert.Equal(t, "sandman", name) - assert.Equal(t, "v2-075-1989", issueNumber) -} - -func TestReadallcomicsGetInfoSandmanDeluxeEdition(t *testing.T) { - opt := &config.Options{ - URL: "https://readallcomics.com/sandman-v2-_the_deluxe_edition-5-part-6-1989/", - All: false, - Last: false, - Debug: false, - Logger: logger.NewLogger(false, make(chan string)), - } - readallcomics := NewReadallcomics(opt) - name, issueNumber := readallcomics.GetInfo("https://readallcomics.com/sandman-v2-_the_deluxe_edition-5-part-6-1989/") - assert.Equal(t, "sandman", name) - assert.Equal(t, "v2-_the_deluxe_edition-5-part-6-1989", issueNumber) -} +const ( + readAllIssuePath = "/sandman-v2-075-1989/" + readAllIssueAlt = "/sandman-v2-_the_deluxe_edition-5-part-6-1989/" + readAllCategory = "/category/sandman/" +) -func TestReadallcomicsRetrieveIssueLinks(t *testing.T) { - opt := &config.Options{ - URL: "https://readallcomics.com/sandman-v2-075-1989/", - All: false, - Last: false, - Debug: false, - Logger: logger.NewLogger(false, make(chan string)), - } - readallcomics := NewReadallcomics(opt) - issues, err := readallcomics.RetrieveIssueLinks() - assert.Nil(t, err) - assert.Equal(t, 1, len(issues)) - assert.Equal(t, "https://readallcomics.com/sandman-v2-075-1989/", issues[0]) +func newReadAllComicsServer() *httptest.Server { + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + base := "http://" + r.Host + switch r.URL.Path { + case readAllIssuePath, readAllIssueAlt: + html := ` + + + + + + + ` + _, _ = w.Write([]byte(html)) + case readAllCategory: + html := ` + + +
      + + +
    + + ` + _, _ = w.Write([]byte(html)) + default: + http.NotFound(w, r) + } + })) } -func TestReadallcomicsRetrieveIssueLinksFromSandmanCategory(t *testing.T) { - opt := &config.Options{ - URL: "http://readallcomics.com/category/sandman/", - All: true, - Last: false, - Debug: false, - Logger: logger.NewLogger(false, make(chan string)), - } - readallcomics := NewReadallcomics(opt) - issues, err := readallcomics.RetrieveIssueLinks() - assert.Nil(t, err) - assert.Greater(t, len(issues), 1) +func TestReadAllComicsScraper(t *testing.T) { + server := newReadAllComicsServer() + defer server.Close() - // Check if our test URLs are present - expectedURLs := []string{ - "https://readallcomics.com/sandman-v2-075-1989/", - "https://readallcomics.com/sandman-v2-_the_deluxe_edition-5-part-6-1989/", + opts := &config.Options{ + URL: server.URL + readAllIssuePath, + Logger: logger.NewLogger(false, nil), } - issueSet := make(map[string]bool) - for _, issue := range issues { - issueSet[issue] = true - } - - for _, expectedURL := range expectedURLs { - assert.True(t, issueSet[expectedURL], "Expected URL %s should be found in issues", expectedURL) - } + scraper := NewReadallcomics(opts) + + comic := &core.Comic{URLSource: server.URL + readAllIssuePath} + require.NoError(t, scraper.Initialize(comic)) + require.Equal(t, []string{ + "https://cdn.example.com/sandman/001.jpg", + "https://cdn.example.com/sandman/002.jpg", + }, comic.Links) + + // Category listing for All + opts.All = true + opts.URL = server.URL + readAllCategory + scraper = NewReadallcomics(opts) + issues, err := scraper.RetrieveIssueLinks() + require.NoError(t, err) + require.Equal(t, []string{ + server.URL + readAllIssuePath, + server.URL + readAllIssueAlt, + }, issues) + + // Last issue from category + opts.All = false + opts.Last = true + scraper = NewReadallcomics(opts) + last, err := scraper.RetrieveIssueLinks() + require.NoError(t, err) + require.Equal(t, []string{server.URL + readAllIssueAlt}, last) } -func TestReadallcomicsRetrieveIssueLinksLastFromSandman(t *testing.T) { - opt := &config.Options{ - URL: "http://readallcomics.com/category/sandman/", - All: false, - Last: true, - Debug: false, - Logger: logger.NewLogger(false, make(chan string)), - } - readallcomics := NewReadallcomics(opt) - issues, err := readallcomics.RetrieveIssueLinks() - assert.Nil(t, err) - assert.Equal(t, 1, len(issues)) - // Should return the last issue from the sandman category -} - -func TestReadallcomicsGetIssuesFromSandmanCategory(t *testing.T) { - opt := &config.Options{ - URL: "http://readallcomics.com/category/sandman/", - All: false, - Last: false, - Debug: false, - Logger: logger.NewLogger(false, make(chan string)), - } - readallcomics := NewReadallcomics(opt) - issues, err := readallcomics.getIssues("http://readallcomics.com/category/sandman/") - assert.Nil(t, err) - assert.Greater(t, len(issues), 0) - - // Check if expected URLs are present in the results - expectedURLs := []string{ - "https://readallcomics.com/sandman-v2-075-1989/", - "https://readallcomics.com/sandman-v2-_the_deluxe_edition-5-part-6-1989/", - } - - issueSet := make(map[string]bool) - for _, issue := range issues { - issueSet[issue] = true +func TestReadAllComicsGetInfoParsing(t *testing.T) { + scraper := NewReadallcomics(&config.Options{}) + + tests := []struct { + url string + expectedName string + expectedIssue string + }{ + {"https://readallcomics.com/something-is-killing-the-children-000-2024/", "something is killing the children", "000-2024"}, + {"https://readallcomics.com/sandman-v2-075-1989/", "sandman", "v2-075-1989"}, + {"https://readallcomics.com/sandman-v2-_the_deluxe_edition-5-part-6-1989/", "sandman", "v2-_the_deluxe_edition-5-part-6-1989"}, } - for _, expectedURL := range expectedURLs { - assert.True(t, issueSet[expectedURL], "Expected URL %s should be found in issues", expectedURL) + for _, tc := range tests { + name, issue := scraper.GetInfo(tc.url) + require.Equal(t, tc.expectedName, name) + require.Equal(t, tc.expectedIssue, issue) } } diff --git a/pkg/sites/readcomiconline_test.go b/pkg/sites/readcomiconline_test.go index 2e573079..b7dbaa57 100644 --- a/pkg/sites/readcomiconline_test.go +++ b/pkg/sites/readcomiconline_test.go @@ -1,94 +1,87 @@ package sites import ( + "fmt" + "net/http" + "net/http/httptest" "testing" "github.com/Girbons/comics-downloader/internal/logger" "github.com/Girbons/comics-downloader/pkg/config" "github.com/Girbons/comics-downloader/pkg/core" - "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) -func TestReadComicOnlineSetup(t *testing.T) { - - comic := new(core.Comic) - comic.URLSource = "https://readcomiconline.li/Comic/Batman-2016/Issue-58?id=143175" +const ( + rcoIssuePath = "/Comic/My-Comic/Issue-2" + rcoListPath = "/Comic/My-Comic" +) - opt := - &config.Options{ - URL: "https://readcomiconline.li/Comic/Batman-2016/Issue-58?id=143175", - All: false, - Last: false, - Debug: false, - Logger: logger.NewLogger(false, make(chan string)), - } - readComicOnline := NewReadComiconline(opt) - err := readComicOnline.Initialize(comic) +func newReadComicOnlineServer() *httptest.Server { + issueHTML := ` + + + + + + ` - assert.Nil(t, err) - assert.Equal(t, 24, len(comic.Links)) -} + listHTML := ` + + + Issue 2 + Issue 1 + + ` -func TestReadComicOnlineGetInfo(t *testing.T) { - opt := - &config.Options{ - URL: "https://readcomiconline.li/Comic/Batman-2016/Issue-58?id=143175", - All: false, - Last: false, - Debug: false, - Logger: logger.NewLogger(false, make(chan string)), + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case rcoIssuePath: + _, _ = fmt.Fprint(w, issueHTML) + case rcoListPath: + _, _ = fmt.Fprint(w, listHTML) + default: + http.NotFound(w, r) } - readComicOnline := NewReadComiconline(opt) - name, issueNumber := readComicOnline.GetInfo("https://readcomiconline.li/Comic/Batman-2016/Issue-58?id=143175") - - assert.Equal(t, "Batman-2016", name) - assert.Equal(t, "58", issueNumber) + })) } -func TestReadComicOnlineRetrieveIssueLinks(t *testing.T) { - opt := - &config.Options{ - URL: "https://readcomiconline.li/Comic/100-Bullets", - All: false, - Last: false, - Debug: false, - Logger: logger.NewLogger(false, make(chan string)), - } - readComicOnline := NewReadComiconline(opt) - issues, err := readComicOnline.RetrieveIssueLinks() +func TestReadComicOnlineScraper(t *testing.T) { + server := newReadComicOnlineServer() + defer server.Close() - assert.Nil(t, err) - assert.Equal(t, 100, len(issues)) -} + originalBase := baseUrl + baseUrl = server.URL + defer func() { baseUrl = originalBase }() -func TestReadComicOnlineRetrieveIssueLinksLastChapter(t *testing.T) { - opt := - &config.Options{ - URL: "https://readcomiconline.li/Comic/100-Bullets", - All: false, - Last: true, - Debug: false, - Logger: logger.NewLogger(false, make(chan string)), - } - readComicOnline := NewReadComiconline(opt) - issues, err := readComicOnline.RetrieveIssueLinks() + opts := &config.Options{ + URL: server.URL + rcoIssuePath, + Logger: logger.NewLogger(false, nil), + } - assert.Nil(t, err) - assert.Equal(t, 1, len(issues)) -} + scraper := NewReadComiconline(opts) -func TestReadComicOnlineRetrieveLastIssueLink(t *testing.T) { - opt := - &config.Options{ - URL: "https://readcomiconline.li/Comic/100-Bullets", - All: false, - Last: true, - Debug: false, - Logger: logger.NewLogger(false, make(chan string)), - } - readComicOnline := NewReadComiconline(opt) - issue, err := readComicOnline.retrieveLastIssue("https://readcomiconline.li/Comic/100-Bullets") + comic := &core.Comic{URLSource: server.URL + rcoIssuePath} + require.NoError(t, scraper.Initialize(comic)) + require.Equal(t, []string{ + "https://2.bp.blogspot.com/abc123=s1600?", + "https://2.bp.blogspot.com/def456=s1600?", + }, comic.Links) + + opts.All = true + opts.URL = server.URL + rcoListPath + scraper = NewReadComiconline(opts) + issues, err := scraper.RetrieveIssueLinks() + require.NoError(t, err) + require.Equal(t, []string{ + server.URL + "/Comic/My-Comic/Issue-2", + server.URL + "/Comic/My-Comic/Issue-1", + }, issues) - assert.Nil(t, err) - assert.Equal(t, "https://readcomiconline.li/Comic/100-Bullets/Issue-100-2", issue) + opts.All = false + opts.Last = true + scraper = NewReadComiconline(opts) + last, err := scraper.RetrieveIssueLinks() + require.NoError(t, err) + require.Equal(t, []string{server.URL + "/Comic/My-Comic/Issue-2"}, last) } From 56da40a8d217694161d113f438ad31600ad2c929 Mon Sep 17 00:00:00 2001 From: hollisticated-horse Date: Tue, 21 Oct 2025 19:28:55 +0200 Subject: [PATCH 12/79] feat: add request throttling --- cmd/downloader/main.go | 8 ++++++++ pkg/config/options.go | 11 +++++++++++ pkg/core/core.go | 27 +++++++++++++++++++++++++++ pkg/core/core_test.go | 15 +++++++++------ 4 files changed, 55 insertions(+), 6 deletions(-) diff --git a/cmd/downloader/main.go b/cmd/downloader/main.go index 59e38402..80232d5d 100644 --- a/cmd/downloader/main.go +++ b/cmd/downloader/main.go @@ -5,6 +5,7 @@ import ( "fmt" "os" "strings" + "time" "github.com/Girbons/comics-downloader/cmd/app" "github.com/Girbons/comics-downloader/internal/version" @@ -46,6 +47,9 @@ var ( // request customization userAgentsCSV string sessionCookie string + // throttling + requestDelay time.Duration + requestDelayJitter time.Duration ) func init() { @@ -68,6 +72,8 @@ func init() { flag.StringVar(&issueFolderName, "issue-folder-name", "issue-", "Folder name where each issue/chapter will be saved, default 'issue-#'") flag.StringVar(&userAgentsCSV, "user-agents", "", "Comma-separated list of alternative User-Agent values to rotate per request") flag.StringVar(&sessionCookie, "session-cookie", "", "Custom Cookie header value (e.g., cf_clearance=...; other=...) for protected sources") + flag.DurationVar(&requestDelay, "request-delay", config.DefaultRequestDelay, "Base delay inserted before downloading each image (e.g. 500ms)") + flag.DurationVar(&requestDelayJitter, "request-delay-jitter", config.DefaultRequestDelayJitter, "Maximum additional random delay added to the base request delay (e.g. 250ms)") flag.IntVar(&daemonTimeout, "daemon-timeout", 600, "DaemonTimeout (seconds), specifies how often the downloader runs") } @@ -93,6 +99,8 @@ func buildOptions() config.Options { IssueFolderName: issueFolderName, UserAgents: splitAndTrim(userAgentsCSV), SessionCookie: strings.TrimSpace(sessionCookie), + RequestDelay: requestDelay, + RequestDelayJitter: requestDelayJitter, } } diff --git a/pkg/config/options.go b/pkg/config/options.go index cdfb092f..b3d64103 100644 --- a/pkg/config/options.go +++ b/pkg/config/options.go @@ -1,10 +1,19 @@ package config import ( + "time" + "github.com/Girbons/comics-downloader/internal/logger" "github.com/Girbons/comics-downloader/pkg/http" ) +const ( + // DefaultRequestDelay defines the base time to wait between subsequent image requests. + DefaultRequestDelay = 500 * time.Millisecond + // DefaultRequestDelayJitter adds up to this much random extra delay to avoid fixed patterns. + DefaultRequestDelayJitter = 250 * time.Millisecond +) + // Options represents the comics downloader options. type Options struct { Debug bool @@ -27,6 +36,8 @@ type Options struct { IssueFolderName string UserAgents []string SessionCookie string + RequestDelay time.Duration + RequestDelayJitter time.Duration Client *http.ComicClient Logger *logger.Logger diff --git a/pkg/core/core.go b/pkg/core/core.go index cc0e8fb4..be790ebc 100644 --- a/pkg/core/core.go +++ b/pkg/core/core.go @@ -5,6 +5,7 @@ import ( "context" "fmt" "image" + "math/rand" "os" "path" "path/filepath" @@ -231,6 +232,19 @@ func (comic *Comic) DownloadImages(options *config.Options) (*DownloadResult, er progress := progressbar.NewOptions(len(comic.Links), progressbar.OptionSetRenderBlankState(true)) format := util.ImageType(comic.ImagesFormat) + requestDelay := options.RequestDelay + requestJitter := options.RequestDelayJitter + if requestDelay < 0 { + requestDelay = 0 + } + if requestJitter < 0 { + requestJitter = 0 + } + if requestDelay == 0 && requestJitter == 0 { + requestDelay = config.DefaultRequestDelay + requestJitter = config.DefaultRequestDelayJitter + } + type downloadJob struct { index int link string @@ -251,6 +265,8 @@ func (comic *Comic) DownloadImages(options *config.Options) (*DownloadResult, er group, ctx := errgroup.WithContext(ctx) sem := semaphore.NewWeighted(int64(runtime.NumCPU())) var mu sync.Mutex + rng := rand.New(rand.NewSource(time.Now().UnixNano())) + var rngMu sync.Mutex for _, job := range jobs { job := job @@ -263,6 +279,17 @@ func (comic *Comic) DownloadImages(options *config.Options) (*DownloadResult, er reqCtx, cancelReq := context.WithTimeout(ctx, 30*time.Second) defer cancelReq() + sleepDuration := requestDelay + if requestJitter > 0 { + rngMu.Lock() + extra := time.Duration(rng.Int63n(int64(requestJitter))) + rngMu.Unlock() + sleepDuration += extra + } + if sleepDuration > 0 { + time.Sleep(sleepDuration) + } + request, err := client.PrepareRequest(job.link, comic.Source) if err != nil { return err diff --git a/pkg/core/core_test.go b/pkg/core/core_test.go index 889341ca..0d2e060d 100644 --- a/pkg/core/core_test.go +++ b/pkg/core/core_test.go @@ -11,6 +11,7 @@ import ( "os" "path/filepath" "testing" + "time" "github.com/Girbons/comics-downloader/internal/logger" "github.com/Girbons/comics-downloader/pkg/config" @@ -41,12 +42,14 @@ func newTestOptions(t *testing.T, server *httptest.Server) *config.Options { ) return &config.Options{ - OutputFolder: t.TempDir(), - CreateDefaultPath: true, - Debug: false, - Logger: logger.NewLogger(false, nil), - Client: client, - IssueFolderName: "issue-", + OutputFolder: t.TempDir(), + CreateDefaultPath: true, + Debug: false, + Logger: logger.NewLogger(false, nil), + Client: client, + IssueFolderName: "issue-", + RequestDelay: time.Nanosecond, + RequestDelayJitter: 0, } } From 48dc7ecf67f3e0745f0651f9ef32472745e118e5 Mon Sep 17 00:00:00 2001 From: hollisticated-horse Date: Thu, 23 Oct 2025 18:17:08 +0200 Subject: [PATCH 13/79] feat(gui): improve download feedback --- cmd/gui/gui.go | 6 +++-- cmd/gui/main.go | 64 +++++++++++++++++++++++++++++++++++++++++-------- 2 files changed, 58 insertions(+), 12 deletions(-) diff --git a/cmd/gui/gui.go b/cmd/gui/gui.go index 364f4982..c3c44873 100644 --- a/cmd/gui/gui.go +++ b/cmd/gui/gui.go @@ -1,6 +1,8 @@ package main import ( + "strings" + "fyne.io/fyne/widget" downloader "github.com/Girbons/comics-downloader/cmd/app" @@ -44,7 +46,7 @@ func (d *Downloader) Submit() { Debug: d.Debug.Checked, All: d.AllChapters.Checked, Last: d.LastChapter.Checked, - URL: d.URL.Text, + URL: strings.TrimSpace(d.URL.Text), Format: d.Format.Selected, Country: d.Country.Text, ImagesFormat: d.ImagesFormat.Selected, @@ -55,5 +57,5 @@ func (d *Downloader) Submit() { CustomComicName: d.CustomComicName.Text, } - downloader.GuiRun(opts) + go downloader.GuiRun(opts) } diff --git a/cmd/gui/main.go b/cmd/gui/main.go index 906f7c2c..f5f658d1 100644 --- a/cmd/gui/main.go +++ b/cmd/gui/main.go @@ -2,6 +2,7 @@ package main import ( "fmt" + "strings" "fyne.io/fyne" "fyne.io/fyne/app" @@ -12,19 +13,54 @@ import ( "github.com/Girbons/comics-downloader/internal/version" ) -func watchLogs(logSection *container.Scroll, box *widget.Box) { - for { - box.Append(widget.NewLabel(<-downloader.Messages)) - logSection.Resize(logSection.Size()) +func watchLogs(logSection *container.Scroll, box *widget.Box, statusLabel *widget.Label) { + for message := range downloader.Messages { + trimmed := strings.TrimSpace(message) + if trimmed == "" { + continue + } + + level := "" + text := trimmed + if parts := strings.SplitN(trimmed, ":", 2); len(parts) == 2 { + level = strings.ToUpper(strings.TrimSpace(parts[0])) + text = strings.TrimSpace(parts[1]) + } + + switch level { + case "INFO", "DEBUG": + if text == "" { + statusLabel.SetText(level) + } else { + statusLabel.SetText(text) + } + case "WARNING", "ERROR": + statusLabel.SetText(trimmed) + box.Append(widget.NewLabel(trimmed)) + logSection.ScrollToBottom() + default: + statusLabel.SetText(trimmed) + } } } -func appStatus(downloadButton *widget.Button) { - for { - if <-downloader.AppStatus { +func appStatus(downloadButton *widget.Button, progress *widget.ProgressBarInfinite, statusLabel *widget.Label) { + for running := range downloader.AppStatus { + if running { downloadButton.Disable() + if !progress.Visible() { + progress.Show() + } + if !progress.Running() { + progress.Start() + } + statusLabel.SetText("Downloading...") } else { + progress.Hide() downloadButton.Enable() + if statusLabel.Text == "Downloading..." { + statusLabel.SetText("Ready") + } } } } @@ -66,6 +102,12 @@ func main() { issuesRange := widget.NewEntry() issuesRange.SetPlaceHolder("1-10") + statusLabel := widget.NewLabel("Ready") + statusLabel.Wrapping = fyne.TextWrapWord + + progress := widget.NewProgressBarInfinite() + progress.Hide() + d := &Downloader{ URL: urlEntry, Country: countryEntry, @@ -116,10 +158,12 @@ func main() { // logSection := widget.NewScrollContainer(box) logSection := container.NewScroll(box) - go watchLogs(logSection, box) - go appStatus(submitButton) + footer := container.NewVBox(statusLabel, progress, buttons) + + go watchLogs(logSection, box, statusLabel) + go appStatus(submitButton, progress, statusLabel) - w.SetContent(fyne.NewContainerWithLayout(layout.NewBorderLayout(form, buttons, nil, nil), form, buttons, logSection)) + w.SetContent(fyne.NewContainerWithLayout(layout.NewBorderLayout(form, footer, nil, nil), form, footer, logSection)) w.Resize(fyne.NewSize(800, 400)) w.ShowAndRun() } From 7021ddb86397466e82017f690897acde9142d2a1 Mon Sep 17 00:00:00 2001 From: hollisticated-horse Date: Sat, 25 Oct 2025 00:58:59 +0200 Subject: [PATCH 14/79] fix: check close errors --- internal/version/version.go | 13 ++++- pkg/core/core.go | 107 +++++++++++++++++++++++++++++++----- pkg/http/client.go | 4 +- pkg/sites/http_helpers.go | 7 ++- pkg/sites/loader.go | 16 +++++- 5 files changed, 127 insertions(+), 20 deletions(-) diff --git a/internal/version/version.go b/internal/version/version.go index 78795aaf..4b46b67f 100644 --- a/internal/version/version.go +++ b/internal/version/version.go @@ -3,7 +3,9 @@ package version import ( "context" "encoding/json" + "errors" "fmt" + "log" "net/http" "sync" "time" @@ -19,6 +21,9 @@ const ( // Tag specifies the current release tag. Overridden at build time via -ldflags. var Tag = "development" +// ErrInvalidSemverTag indicates that either the current or latest tag is not valid semantic version. +var ErrInvalidSemverTag = errors.New("invalid semver tag") + type release struct { TagName string `json:"tag_name"` HTMLURL string `json:"html_url"` @@ -64,7 +69,11 @@ func IsNewAvailable(ctx context.Context, client *http.Client) (bool, string, err updateCache(false, "", err) return false, "", err } - defer resp.Body.Close() + defer func() { + if closeErr := resp.Body.Close(); closeErr != nil { + log.Printf("version: failed to close release response body: %v", closeErr) + } + }() if resp.StatusCode != http.StatusOK { err = fmt.Errorf("unexpected status code: %d", resp.StatusCode) @@ -85,7 +94,7 @@ func IsNewAvailable(ctx context.Context, client *http.Client) (bool, string, err latest := releases[0] if !semver.IsValid(Tag) || !semver.IsValid(latest.TagName) { - err = fmt.Errorf("invalid semver tag (current=%q latest=%q)", Tag, latest.TagName) + err = fmt.Errorf("%w (current=%q latest=%q)", ErrInvalidSemverTag, Tag, latest.TagName) updateCache(false, "", err) return false, "", err } diff --git a/pkg/core/core.go b/pkg/core/core.go index be790ebc..e03051e1 100644 --- a/pkg/core/core.go +++ b/pkg/core/core.go @@ -3,9 +3,12 @@ package core import ( "bytes" "context" + "encoding/base64" "fmt" "image" + "io" "math/rand" + "net/http" "os" "path" "path/filepath" @@ -125,7 +128,9 @@ func (comic *Comic) makePDF(options *config.Options, images *DownloadResult) err } } else { im, _, err := image.DecodeConfig(img) - img.Close() + if closeErr := img.Close(); closeErr != nil && options.Logger != nil { + options.Logger.Errorf("failed to close image %s: %v", fileName, closeErr) + } if err != nil { if options.Logger != nil { options.Logger.Error(err.Error()) @@ -177,7 +182,13 @@ func (comic *Comic) makeCBRZ(options *config.Options, images *DownloadResult) er if err != nil { return err } - defer out.Close() + defer func() { + if out != nil { + if closeErr := out.Close(); closeErr != nil && options.Logger != nil { + options.Logger.Errorf("failed to close archive %s: %v", zipArchiveName, closeErr) + } + } + }() fileMap := make(map[string]string) for _, filePath := range images.FilePaths { @@ -194,6 +205,11 @@ func (comic *Comic) makeCBRZ(options *config.Options, images *DownloadResult) er return err } + if err = out.Close(); err != nil { + return err + } + out = nil + if err = os.Rename(zipArchiveName, newName); err != nil { return err } @@ -267,6 +283,7 @@ func (comic *Comic) DownloadImages(options *config.Options) (*DownloadResult, er var mu sync.Mutex rng := rand.New(rand.NewSource(time.Now().UnixNano())) var rngMu sync.Mutex + const sniffLimit = 256 for _, job := range jobs { job := job @@ -275,6 +292,11 @@ func (comic *Comic) DownloadImages(options *config.Options) (*DownloadResult, er } group.Go(func() error { defer sem.Release(1) + defer func() { + if progressErr := progress.Add(1); progressErr != nil && options.Logger != nil { + options.Logger.Error(progressErr.Error()) + } + }() reqCtx, cancelReq := context.WithTimeout(ctx, 30*time.Second) defer cancelReq() @@ -300,7 +322,54 @@ func (comic *Comic) DownloadImages(options *config.Options) (*DownloadResult, er if err != nil { return err } - defer response.Body.Close() + defer func() { + if closeErr := response.Body.Close(); closeErr != nil { + if options.Logger != nil { + options.Logger.Errorf("failed to close response body for %s: %v", job.link, closeErr) + } + } + }() + + if response.StatusCode < http.StatusOK || response.StatusCode >= http.StatusMultipleChoices { + if options.Logger != nil { + options.Logger.Errorf("There was an error while downloading image number: %d - comic issue: %s (status code: %d)", job.index, comic.IssueNumber, response.StatusCode) + } + return nil + } + + data, err := io.ReadAll(response.Body) + if err != nil { + if options.Logger != nil { + options.Logger.Errorf("Failed reading image number: %d - comic issue: %s (%v)", job.index, comic.IssueNumber, err) + } + return nil + } + + if len(data) == 0 { + if options.Logger != nil { + options.Logger.Errorf("Image number: %d - comic issue: %s returned an empty response body", job.index, comic.IssueNumber) + } + return nil + } + + contentType := strings.ToLower(strings.TrimSpace(response.Header.Get("Content-Type"))) + if contentType == "" { + sniffLen := len(data) + if sniffLen > sniffLimit { + sniffLen = sniffLimit + } + contentType = strings.ToLower(http.DetectContentType(data[:sniffLen])) + } + + isWebp := strings.HasSuffix(strings.ToLower(job.link), ".webp") || strings.Contains(contentType, "image/webp") + if options.Logger != nil && contentType != "" && !strings.HasPrefix(contentType, "image/") { + reportLen := len(data) + if reportLen > sniffLimit { + reportLen = sniffLimit + } + snippet := base64.StdEncoding.EncodeToString(data[:reportLen]) + options.Logger.Errorf("Unexpected content type '%s' while downloading image number: %d - url: %s (bytes=%d, snippet_base64=%s)", contentType, job.index, job.link, len(data), snippet) + } fileName := fmt.Sprintf("%04d-image.%s", job.index, format) targetPath := filepath.Join(dir, fileName) @@ -309,23 +378,31 @@ func (comic *Comic) DownloadImages(options *config.Options) (*DownloadResult, er return err } - isWebp := strings.HasSuffix(strings.ToLower(job.link), ".webp") - if err := util.SaveImage(imgFile, response.Body, format, isWebp); err != nil { + reader := bytes.NewReader(data) + if err := util.SaveImage(imgFile, reader, format, isWebp); err != nil { if options.Logger != nil { - options.Logger.Errorf("There was an error while downloading image number: %d - comic issue: %s (%v)", job.index, comic.IssueNumber, err) + reportLen := len(data) + if reportLen > sniffLimit { + reportLen = sniffLimit + } + snippet := base64.StdEncoding.EncodeToString(data[:reportLen]) + options.Logger.Errorf("There was an error while downloading image number: %d - comic issue: %s (%v) (content-type=%s bytes=%d snippet_base64=%s)", job.index, comic.IssueNumber, err, contentType, len(data), snippet) + } + if closeErr := imgFile.Close(); closeErr != nil && options.Logger != nil { + options.Logger.Errorf("failed to close image file %s: %v", targetPath, closeErr) + } + if removeErr := os.Remove(targetPath); removeErr != nil && options.Logger != nil { + options.Logger.Errorf("failed to remove incomplete image %s: %v", targetPath, removeErr) } - imgFile.Close() - _ = os.Remove(targetPath) } else { - imgFile.Close() + if closeErr := imgFile.Close(); closeErr != nil && options.Logger != nil { + options.Logger.Errorf("failed to close image file %s: %v", targetPath, closeErr) + } mu.Lock() results[job.index] = targetPath mu.Unlock() } - if progressErr := progress.Add(1); progressErr != nil && options.Logger != nil { - options.Logger.Error(progressErr.Error()) - } return nil }) } @@ -371,7 +448,11 @@ func (comic *Comic) MakeComic(options *config.Options) error { if err != nil { return err } - defer os.RemoveAll(result.Dir) + defer func() { + if err := os.RemoveAll(result.Dir); err != nil && options.Logger != nil { + options.Logger.Errorf("failed to remove temporary directory %s: %v", result.Dir, err) + } + }() switch comic.Format { case EPUB: diff --git a/pkg/http/client.go b/pkg/http/client.go index 2b84218e..87c47d28 100644 --- a/pkg/http/client.go +++ b/pkg/http/client.go @@ -197,7 +197,9 @@ func (c *ComicClient) Do(req *http.Request) (*http.Response, error) { if resp.StatusCode >= 500 { lastErr = fmt.Errorf("server error: %d", resp.StatusCode) - resp.Body.Close() + if closeErr := resp.Body.Close(); closeErr != nil { + lastErr = fmt.Errorf("%w; close error: %v", lastErr, closeErr) + } continue } diff --git a/pkg/sites/http_helpers.go b/pkg/sites/http_helpers.go index 5347b527..0b4d202b 100644 --- a/pkg/sites/http_helpers.go +++ b/pkg/sites/http_helpers.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" "io" + "log" "net/http" urlpkg "net/url" @@ -67,7 +68,11 @@ func fetchBytes(ctx context.Context, client *httpclient.ComicClient, link string if err != nil { return nil, err } - defer resp.Body.Close() + defer func() { + if closeErr := resp.Body.Close(); closeErr != nil { + log.Printf("sites: failed to close response body for %s: %v", link, closeErr) + } + }() if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices { return nil, fmt.Errorf("unexpected status code %d for %s", resp.StatusCode, link) diff --git a/pkg/sites/loader.go b/pkg/sites/loader.go index 4125853b..685c79b0 100644 --- a/pkg/sites/loader.go +++ b/pkg/sites/loader.go @@ -1,7 +1,6 @@ package sites import ( - "errors" "fmt" "regexp" "strconv" @@ -18,7 +17,7 @@ func initializeCollection(issues []string, options *config.Options, base BaseSit var err error if len(issues) == 0 { - return collection, errors.New("No issues found") + return collection, fmt.Errorf("no issues found for URL %q; ensure it points to a specific comic or chapter page", options.URL) } var startRange, endRange float64 @@ -43,7 +42,10 @@ func initializeCollection(issues []string, options *config.Options, base BaseSit continue } - dir, _ := util.PathSetup(options.CreateDefaultPath, options.OutputFolder, options.Source, name) + dir, pathErr := util.PathSetup(options.CreateDefaultPath, options.OutputFolder, options.Source, name) + if pathErr != nil { + return collection, pathErr + } fileName := util.GetPathToFile(dir, name, issueNumber, options.Format, options.IssueNumberNameOnly) if util.DirectoryOrFileDoesNotExist(fileName) || options.ImagesOnly { @@ -116,10 +118,18 @@ func LoadComicFromSource(options *config.Options) ([]*core.Comic, error) { return collection, err } + if options.Logger != nil && options.Debug { + options.Logger.Debugf("sites: retrieving issues for %s", options.URL) + } + issues, err = base.RetrieveIssueLinks() if err != nil { return collection, err } + if options.Logger != nil && options.Debug { + options.Logger.Debugf("sites: %d issue(s) discovered for %s", len(issues), options.URL) + } + return initializeCollection(issues, options, base) } From 00ea7530ecea3420f6da9a0af747f7f20d516ff3 Mon Sep 17 00:00:00 2001 From: hollisticated_horse Date: Tue, 3 Feb 2026 15:39:15 +0100 Subject: [PATCH 15/79] fix(readallcomics): improve chapter/issue URL parsing and error handling - Add error check when finding chapter list - Normalize URLs with proper trimming and validation - Add fallback for missing select box - Add empty chapters validation - Fix variable naming (issueUrl -> issueURL) --- pkg/sites/readallcomics.go | 90 ++++++++++++++++++++++++++++++++------ 1 file changed, 76 insertions(+), 14 deletions(-) diff --git a/pkg/sites/readallcomics.go b/pkg/sites/readallcomics.go index 678ea349..c1795284 100644 --- a/pkg/sites/readallcomics.go +++ b/pkg/sites/readallcomics.go @@ -36,9 +36,12 @@ func (r *Readallcomics) retrieveImageLinks(comic *core.Comic) ([]string, error) images := document.FindAll("img") for _, img := range images { - url := img.Attrs()["src"] - if util.IsURLValid(url) { - links = append(links, url) + src, ok := img.Attrs()["src"] + if !ok { + continue + } + if util.IsURLValid(src) && !util.IsValueInSlice(src, links) { + links = append(links, src) } } @@ -61,22 +64,77 @@ func (r *Readallcomics) getIssues(url string) ([]string, error) { doc := soup.HTMLParse(response) if strings.Contains(url, "category") { - chapters := doc.Find("ul", "class", "list-story").FindAll("a") - for _, chapter := range chapters { - issueUrl := chapter.Attrs()["href"] - if util.IsURLValid(issueUrl) { - links = append(links, issueUrl) - } + chapterList := doc.Find("ul", "class", "list-story") + if chapterList.Error != nil { + return nil, fmt.Errorf("readallcomics: unable to find chapter list on %s", url) + } + for _, chapter := range chapterList.FindAll("a") { + issueURL, ok := chapter.Attrs()["href"] + if !ok { + continue + } + issueURL = strings.TrimSpace(issueURL) + if issueURL == "" { + continue + } + if !strings.HasPrefix(issueURL, "http") { + issueURL = fmt.Sprintf("%s/%s", strings.TrimRight(DefaultUrl, "/"), strings.TrimLeft(issueURL, "/")) + } + if util.IsURLValid(issueURL) && !util.IsValueInSlice(issueURL, links) { + links = append(links, issueURL) + } } } else { - chapters := doc.Find("select", "id", "selectbox").FindAll("option") - for _, chapter := range chapters { - issueUrl := chapter.Attrs()["value"] - if util.IsURLValid(issueUrl) { - links = append(links, issueUrl) + selectBox := doc.Find("select", "id", "selectbox") + if selectBox.Error == nil { + for _, chapter := range selectBox.FindAll("option") { + issueURL, ok := chapter.Attrs()["value"] + if !ok { + continue + } + issueURL = strings.TrimSpace(issueURL) + if issueURL == "" { + continue + } + if !strings.HasPrefix(issueURL, "http") { + issueURL = fmt.Sprintf("%s/%s", strings.TrimRight(DefaultUrl, "/"), strings.TrimLeft(issueURL, "/")) + } + if util.IsURLValid(issueURL) && !util.IsValueInSlice(issueURL, links) { + links = append(links, issueURL) + } } } + + if len(links) == 0 { + // fallback: scan for anchors that point to issues + for _, chapter := range doc.FindAll("a") { + issueURL, ok := chapter.Attrs()["href"] + if !ok { + continue + } + issueURL = strings.TrimSpace(issueURL) + if issueURL == "" { + continue + } + if !strings.HasPrefix(issueURL, "http") { + issueURL = fmt.Sprintf("%s/%s", strings.TrimRight(DefaultUrl, "/"), strings.TrimLeft(issueURL, "/")) + } + if !strings.Contains(issueURL, DefaultUrl) { + continue + } + if strings.Contains(issueURL, "/category/") { + continue + } + if util.IsURLValid(issueURL) && !util.IsValueInSlice(issueURL, links) { + links = append(links, issueURL) + } + } + } + + if len(links) == 0 { + return nil, fmt.Errorf("readallcomics: unable to find issue references on %s", url) + } } if r.options.Debug { @@ -103,6 +161,10 @@ func (r *Readallcomics) RetrieveIssueLinks() ([]string, error) { return nil, err } + if len(chapters) == 0 { + return nil, fmt.Errorf("readallcomics: no chapters found at %s", url) + } + if r.options.Last { return []string{chapters[len(chapters)-1]}, nil } From 619b00b5029ac1f1dafd12a7b136c28cd1a60300 Mon Sep 17 00:00:00 2001 From: hollisticated_horse Date: Tue, 3 Feb 2026 15:39:15 +0100 Subject: [PATCH 16/79] fix(readcomiconline): enhance debug logging for image scraping - Add detailed debug logging with base64-encoded snippets - Log fetch URL and request failures - Show obfuscated entry count and decoded link preview - Limit snippet size to prevent log overflow --- pkg/sites/readcomiconline.go | 32 +++++++++++++++++++++++++++++--- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/pkg/sites/readcomiconline.go b/pkg/sites/readcomiconline.go index d0602ae4..8a295979 100644 --- a/pkg/sites/readcomiconline.go +++ b/pkg/sites/readcomiconline.go @@ -63,12 +63,21 @@ func deobfuscateUrl(imageLink string) (string, error) { func (c *ReadComicOnline) retrieveImageLinks(comic *core.Comic) ([]string, error) { var links []string + const debugSnippetLimit = 4096 comic.URLSource = strings.Split(comic.URLSource, "?")[0] + fetchURL := comic.URLSource + "?quality=hd&readType=1" - response, err := soup.Get(comic.URLSource + "?quality=hd&readType=1") + if c.options.Debug && c.options.Logger != nil { + c.options.Logger.Debugf("readcomiconline: fetching %s", fetchURL) + } + + response, err := soup.Get(fetchURL) if err != nil { - return nil, err + if c.options.Logger != nil { + c.options.Logger.Errorf("readcomiconline: request to %s failed: %v", fetchURL, err) + } + return nil, fmt.Errorf("readcomiconline: fetch %s: %w", fetchURL, err) } re := regexp.MustCompile(`push\(\'(.*?)\'\)`) @@ -87,7 +96,24 @@ func (c *ReadComicOnline) retrieveImageLinks(comic *core.Comic) ([]string, error } } - if c.options.Debug { + if c.options.Debug && c.options.Logger != nil { + c.options.Logger.Debugf("readcomiconline: found %d obfuscated entries, %d valid links for %s", len(match), len(links), comic.URLSource) + snippet := response + if len(snippet) > debugSnippetLimit { + snippet = snippet[:debugSnippetLimit] + } + encoded := base64.StdEncoding.EncodeToString([]byte(snippet)) + c.options.Logger.Debugf("readcomiconline: response snippet (base64, trimmed to %d bytes) = %s", len(snippet), encoded) + if len(match) > 0 { + c.options.Logger.Debugf("readcomiconline: first obfuscated entry (base64) = %s", base64.StdEncoding.EncodeToString([]byte(match[0][1]))) + } + if len(links) > 0 { + preview := links[0] + if len(preview) > 256 { + preview = preview[:256] + "..." + } + c.options.Logger.Debugf("readcomiconline: first decoded link = %s", preview) + } c.options.Logger.Debug(fmt.Sprintf("Image Links found: %s", strings.Join(links, " "))) } From d7a6c61e21f72405cedbc7665172ad2ea7ce6784 Mon Sep 17 00:00:00 2001 From: ProjectDislocate <173194962+ProjectDislocate@users.noreply.github.com> Date: Thu, 5 Mar 2026 22:07:40 -0500 Subject: [PATCH 17/79] fix(app): improve version check error handling thx hollisticated-horse --- cmd/app/downloader.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/cmd/app/downloader.go b/cmd/app/downloader.go index 11af2a38..54a5d16b 100644 --- a/cmd/app/downloader.go +++ b/cmd/app/downloader.go @@ -2,6 +2,7 @@ package app import ( "context" + "errors" "fmt" "os" "strings" @@ -126,7 +127,11 @@ func (r *Runner) download(base config.Options) { isNewVersionAvailable, newVersionLink, err := version.IsNewAvailable(ctx, opts.Client.HTTPClient()) if err != nil { - opts.Logger.Error("There was an error while checking for a new comics-downloader version") + if errors.Is(err, version.ErrInvalidSemverTag) { + opts.Logger.Debugf("Skipping version check: %v", err) + } else { + opts.Logger.Errorf("There was an error while checking for a new comics-downloader version: %v", err) + } } if isNewVersionAvailable { From a84792f079fabb1b5198cd93379eb8429cc670da Mon Sep 17 00:00:00 2001 From: ProjectDislocate <173194962+ProjectDislocate@users.noreply.github.com> Date: Thu, 5 Mar 2026 22:34:28 -0500 Subject: [PATCH 18/79] chore: rename variables for better clarity --- cmd/app/downloader.go | 2 +- pkg/config/options.go | 2 +- pkg/core/core.go | 35 ++++++++++++++++------------ pkg/core/core_test.go | 22 +++++++++--------- pkg/sites/base.go | 2 +- pkg/sites/comicextra.go | 8 +++---- pkg/sites/comicextra_test.go | 7 +++++- pkg/sites/common.go | 4 ++-- pkg/sites/loader.go | 38 ++++++++++++++++--------------- pkg/sites/loader_test.go | 18 +++++++-------- pkg/sites/mangadex.go | 4 ++-- pkg/sites/mangadex_test.go | 14 +++++++----- pkg/sites/mangakakalot.go | 2 +- pkg/sites/mangakakalot_test.go | 10 ++++---- pkg/sites/manganato.go | 2 +- pkg/sites/manganato_test.go | 10 ++++---- pkg/sites/mangareader.go | 8 +++---- pkg/sites/mangareader_test.go | 4 +++- pkg/sites/mangatown.go | 8 +++---- pkg/sites/mangatown_test.go | 4 +++- pkg/sites/readallcomics.go | 6 ++--- pkg/sites/readallcomics_test.go | 4 +++- pkg/sites/readcomiconline.go | 10 ++++---- pkg/sites/readcomiconline_test.go | 4 +++- 24 files changed, 127 insertions(+), 101 deletions(-) diff --git a/cmd/app/downloader.go b/cmd/app/downloader.go index 54a5d16b..910cf2bb 100644 --- a/cmd/app/downloader.go +++ b/cmd/app/downloader.go @@ -151,7 +151,7 @@ func (r *Runner) download(base config.Options) { // check if the link is supported source, check, isDisabled := detector.DetectComic(trimmedURL) - perURL.Source = source + perURL.SourceName = source if !check { perURL.Logger.Error("This site is not supported") diff --git a/pkg/config/options.go b/pkg/config/options.go index b3d64103..e40a6e5f 100644 --- a/pkg/config/options.go +++ b/pkg/config/options.go @@ -31,7 +31,7 @@ type Options struct { CreateDefaultPath bool IssueNumberNameOnly bool URL string - Source string + SourceName string IssuesRange string IssueFolderName string UserAgents []string diff --git a/pkg/core/core.go b/pkg/core/core.go index e03051e1..a05152e5 100644 --- a/pkg/core/core.go +++ b/pkg/core/core.go @@ -41,16 +41,21 @@ const ( PDF = "pdf" ) -// Comic struct contains all the informations about a comic -type Comic struct { +type ComicSource struct { + Name string + URL string +} + +// ComicIssue struct contains all the informations about a comic +type ComicIssue struct { Author string Name string IssueNumber string - Source string - URLSource string Links []string Format string ImagesFormat string + + Source *ComicSource } // DownloadResult captures the outcome of downloading a comic's images. @@ -67,7 +72,7 @@ func ensureClient(options *config.Options) *httpclient.ComicClient { } // makeEPUB creates the epub file. -func (comic *Comic) makeEPUB(options *config.Options, images *DownloadResult) error { +func (comic *ComicIssue) makeEPUB(options *config.Options, images *DownloadResult) error { isCoverSet := false imgTag := `Cover Image` e := epub.NewEpub(comic.IssueNumber) @@ -93,7 +98,7 @@ func (comic *Comic) makeEPUB(options *config.Options, images *DownloadResult) er } } - dir, err := util.PathSetup(options.CreateDefaultPath, options.OutputFolder, comic.Source, comic.Name) + dir, err := util.PathSetup(options.CreateDefaultPath, options.OutputFolder, comic.Source.Name, comic.Name) if err != nil { return err } @@ -109,7 +114,7 @@ func (comic *Comic) makeEPUB(options *config.Options, images *DownloadResult) er } // makePDF create the pdf file. -func (comic *Comic) makePDF(options *config.Options, images *DownloadResult) error { +func (comic *ComicIssue) makePDF(options *config.Options, images *DownloadResult) error { var mmWd, mmHt float64 const px2mm = 0.2645833333 @@ -152,7 +157,7 @@ func (comic *Comic) makePDF(options *config.Options, images *DownloadResult) err pdf.ImageOptions(path.Base(fileName), 0, 0, mmWd, mmHt, false, imageOptions, 0, "") } - dir, err := util.PathSetup(options.CreateDefaultPath, options.OutputFolder, comic.Source, comic.Name) + dir, err := util.PathSetup(options.CreateDefaultPath, options.OutputFolder, comic.Source.Name, comic.Name) if err != nil { return err } @@ -169,8 +174,8 @@ func (comic *Comic) makePDF(options *config.Options, images *DownloadResult) err } // makeCBRZ will create the CBR/CBZ. -func (comic *Comic) makeCBRZ(options *config.Options, images *DownloadResult) error { - dir, err := util.PathSetup(options.CreateDefaultPath, options.OutputFolder, comic.Source, comic.Name) +func (comic *ComicIssue) makeCBRZ(options *config.Options, images *DownloadResult) error { + dir, err := util.PathSetup(options.CreateDefaultPath, options.OutputFolder, comic.Source.Name, comic.Name) if err != nil { return err } @@ -221,14 +226,14 @@ func (comic *Comic) makeCBRZ(options *config.Options, images *DownloadResult) er } // DownloadImages will download the comic/manga images. -func (comic *Comic) DownloadImages(options *config.Options) (*DownloadResult, error) { +func (comic *ComicIssue) DownloadImages(options *config.Options) (*DownloadResult, error) { if len(comic.Links) == 0 { - return nil, fmt.Errorf("download failed, no links found for: %s", comic.URLSource) + return nil, fmt.Errorf("download failed, no links found for: %s", comic.Source.URL) } client := ensureClient(options) - dir, err := util.ImagesPathSetup(options.CreateDefaultPath, options.OutputFolder, comic.Source, comic.Name, options.IssueFolderName, comic.IssueNumber) + dir, err := util.ImagesPathSetup(options.CreateDefaultPath, options.OutputFolder, comic.Source.Name, comic.Name, options.IssueFolderName, comic.IssueNumber) if err != nil { return nil, err } @@ -312,7 +317,7 @@ func (comic *Comic) DownloadImages(options *config.Options) (*DownloadResult, er time.Sleep(sleepDuration) } - request, err := client.PrepareRequest(job.link, comic.Source) + request, err := client.PrepareRequest(job.link, comic.Source.Name) if err != nil { return err } @@ -443,7 +448,7 @@ func filterEmpty(items []string) []string { } // MakeComic will create the file based on the output format selected. -func (comic *Comic) MakeComic(options *config.Options) error { +func (comic *ComicIssue) MakeComic(options *config.Options) error { result, err := comic.DownloadImages(options) if err != nil { return err diff --git a/pkg/core/core_test.go b/pkg/core/core_test.go index 0d2e060d..e729fb92 100644 --- a/pkg/core/core_test.go +++ b/pkg/core/core_test.go @@ -67,9 +67,9 @@ func TestDownloadImagesCreatesFiles(t *testing.T) { opts := newTestOptions(t, server) - comic := &Comic{ + comic := &ComicIssue{ Name: "foo", - Source: "test-source", + Source: &ComicSource{Name: "test-source", URL: server.URL}, IssueNumber: "1", ImagesFormat: "png", Links: buildLinks(server, 3), @@ -95,9 +95,9 @@ func TestMakeComicPDF(t *testing.T) { opts := newTestOptions(t, server) - comic := &Comic{ + comic := &ComicIssue{ Name: "foo", - Source: "test-source", + Source: &ComicSource{Name: "test-source", URL: server.URL}, IssueNumber: "1", Format: PDF, ImagesFormat: "png", @@ -106,7 +106,7 @@ func TestMakeComicPDF(t *testing.T) { require.NoError(t, comic.MakeComic(opts)) - output := filepath.Join(opts.OutputFolder, "comics", comic.Source, comic.Name, "foo-1.pdf") + output := filepath.Join(opts.OutputFolder, "comics", comic.Source.Name, comic.Name, "foo-1.pdf") require.FileExists(t, output) } @@ -116,9 +116,9 @@ func TestMakeComicEPUB(t *testing.T) { opts := newTestOptions(t, server) - comic := &Comic{ + comic := &ComicIssue{ Name: "bar", - Source: "test-source", + Source: &ComicSource{Name: "test-source", URL: server.URL}, IssueNumber: "42", Author: "Author", Format: EPUB, @@ -128,7 +128,7 @@ func TestMakeComicEPUB(t *testing.T) { require.NoError(t, comic.MakeComic(opts)) - output := filepath.Join(opts.OutputFolder, "comics", comic.Source, comic.Name, "bar-42.epub") + output := filepath.Join(opts.OutputFolder, "comics", comic.Source.Name, comic.Name, "bar-42.epub") require.FileExists(t, output) } @@ -138,9 +138,9 @@ func TestMakeComicCBZ(t *testing.T) { opts := newTestOptions(t, server) - comic := &Comic{ + comic := &ComicIssue{ Name: "baz", - Source: "test-source", + Source: &ComicSource{Name: "test-source", URL: server.URL}, IssueNumber: "7", Format: CBZ, ImagesFormat: "png", @@ -149,7 +149,7 @@ func TestMakeComicCBZ(t *testing.T) { require.NoError(t, comic.MakeComic(opts)) - output := filepath.Join(opts.OutputFolder, "comics", comic.Source, comic.Name, "baz-7.cbz") + output := filepath.Join(opts.OutputFolder, "comics", comic.Source.Name, comic.Name, "baz-7.cbz") require.FileExists(t, output) } diff --git a/pkg/sites/base.go b/pkg/sites/base.go index b7bea99b..eebfa94f 100644 --- a/pkg/sites/base.go +++ b/pkg/sites/base.go @@ -6,7 +6,7 @@ import "github.com/Girbons/comics-downloader/pkg/core" // to retrieve a manga/comic basics info and imges links type BaseSite interface { // Initialize will initialize the comic struct with the images link - Initialize(comic *core.Comic) error + Initialize(comic *core.ComicIssue) error // GetInfo will return the comic name and issue number GetInfo(url string) (string, string) diff --git a/pkg/sites/comicextra.go b/pkg/sites/comicextra.go index a31f8228..7f514d19 100644 --- a/pkg/sites/comicextra.go +++ b/pkg/sites/comicextra.go @@ -24,10 +24,10 @@ func NewComicextra(options *config.Options) *Comicextra { } } -func (c *Comicextra) retrieveImageLinks(comic *core.Comic) ([]string, error) { +func (c *Comicextra) retrieveImageLinks(comic *core.ComicIssue) ([]string, error) { var links []string - response, err := soup.Get(comic.URLSource) + response, err := soup.Get(comic.Source.URL) if err != nil { return nil, err } @@ -103,7 +103,7 @@ func (c *Comicextra) RetrieveIssueLinks() ([]string, error) { } if c.options.All && c.isSingleIssue(url) { - url = "https://" + c.options.Source + "/comic/" + comicName + url = "https://" + c.options.SourceName + "/comic/" + comicName } else if c.isSingleIssue(url) { if !strings.HasSuffix(url, "/full") { @@ -160,7 +160,7 @@ func (c *Comicextra) GetInfo(url string) (string, string) { // Initialize will initialize the comic based // on comicextra.com -func (c *Comicextra) Initialize(comic *core.Comic) error { +func (c *Comicextra) Initialize(comic *core.ComicIssue) error { links, err := c.retrieveImageLinks(comic) comic.Links = links diff --git a/pkg/sites/comicextra_test.go b/pkg/sites/comicextra_test.go index 16feda9a..f7fde77c 100644 --- a/pkg/sites/comicextra_test.go +++ b/pkg/sites/comicextra_test.go @@ -69,7 +69,12 @@ func TestComicExtraScraper(t *testing.T) { comicextra := NewComicextra(opts) - comic := &core.Comic{URLSource: server.URL + comicExtraIssueFullPath} + comic := &core.ComicIssue{ + Source: &core.ComicSource{ + Name: "test-source", + URL: server.URL + comicExtraIssueFullPath, + }, + } require.NoError(t, comicextra.Initialize(comic)) require.Equal(t, []string{ "https://cdn.example.com/batman?page=1", diff --git a/pkg/sites/common.go b/pkg/sites/common.go index 06d22a9a..ba8caa42 100644 --- a/pkg/sites/common.go +++ b/pkg/sites/common.go @@ -40,8 +40,8 @@ func MangaKakalotGetInfo(domain string, url string) (string, string) { return name, issueNumber } -func MangaKakalotInitialize(comic *core.Comic) error { - res, err := soup.Get(comic.URLSource) +func MangaKakalotInitialize(comic *core.ComicIssue) error { + res, err := soup.Get(comic.Source.URL) if err != nil { return err } diff --git a/pkg/sites/loader.go b/pkg/sites/loader.go index 685c79b0..a7a46427 100644 --- a/pkg/sites/loader.go +++ b/pkg/sites/loader.go @@ -12,8 +12,8 @@ import ( "github.com/Girbons/comics-downloader/pkg/util" ) -func initializeCollection(issues []string, options *config.Options, base BaseSite) ([]*core.Comic, error) { - var collection []*core.Comic +func initializeCollection(issues []string, options *config.Options, base BaseSite) ([]*core.ComicIssue, error) { + var collection []*core.ComicIssue var err error if len(issues) == 0 { @@ -42,18 +42,20 @@ func initializeCollection(issues []string, options *config.Options, base BaseSit continue } - dir, pathErr := util.PathSetup(options.CreateDefaultPath, options.OutputFolder, options.Source, name) + dir, pathErr := util.PathSetup(options.CreateDefaultPath, options.OutputFolder, options.SourceName, name) if pathErr != nil { return collection, pathErr } fileName := util.GetPathToFile(dir, name, issueNumber, options.Format, options.IssueNumberNameOnly) if util.DirectoryOrFileDoesNotExist(fileName) || options.ImagesOnly { - comic := &core.Comic{ - Name: name, - IssueNumber: issueNumber, - URLSource: url, - Source: options.Source, + comic := &core.ComicIssue{ + Name: name, + IssueNumber: issueNumber, + Source: &core.ComicSource{ + Name: options.SourceName, + URL: url, + }, Format: options.Format, ImagesFormat: options.ImagesFormat, } @@ -88,30 +90,30 @@ func notInIssuesRange(issueNumber string, start, end float64) bool { } // LoadComicFromSource will return an `comic` instance initialized based on the source -func LoadComicFromSource(options *config.Options) ([]*core.Comic, error) { +func LoadComicFromSource(options *config.Options) ([]*core.ComicIssue, error) { var ( base BaseSite issues []string - collection []*core.Comic + collection []*core.ComicIssue err error ) switch { - case strings.Contains(options.Source, "readcomiconline"): + case strings.Contains(options.SourceName, "readcomiconline"): base = NewReadComiconline(options) - case strings.Contains(options.Source, "comicextra"): + case strings.Contains(options.SourceName, "comicextra"): base = NewComicextra(options) - case strings.Contains(options.Source, "mangareader"): + case strings.Contains(options.SourceName, "mangareader"): base = NewMangareader(options) - case strings.Contains(options.Source, "mangatown"): + case strings.Contains(options.SourceName, "mangatown"): base = NewMangatown(options) - case strings.Contains(options.Source, "mangadex"): + case strings.Contains(options.SourceName, "mangadex"): base = NewMangadex(options) - case strings.Contains(options.Source, "readallcomics"): + case strings.Contains(options.SourceName, "readallcomics"): base = NewReadallcomics(options) - case strings.Contains(options.Source, "mangakakalot"): + case strings.Contains(options.SourceName, "mangakakalot"): base = NewMangaKakalot(options) - case strings.Contains(options.Source, "manganato"): + case strings.Contains(options.SourceName, "manganato"): base = NewManganato(options) default: err = fmt.Errorf("source unknown") diff --git a/pkg/sites/loader_test.go b/pkg/sites/loader_test.go index 6204740d..833c04dc 100644 --- a/pkg/sites/loader_test.go +++ b/pkg/sites/loader_test.go @@ -11,11 +11,11 @@ import ( type stubSite struct { issues []string - comics map[string]*core.Comic + comics map[string]*core.ComicIssue } -func (s *stubSite) Initialize(comic *core.Comic) error { - if stub, ok := s.comics[comic.URLSource]; ok { +func (s *stubSite) Initialize(comic *core.ComicIssue) error { + if stub, ok := s.comics[comic.Source.URL]; ok { *comic = *stub return nil } @@ -35,7 +35,7 @@ func (s *stubSite) RetrieveIssueLinks() ([]string, error) { func TestInitializeCollectionFiltersIssues(t *testing.T) { options := &config.Options{ - Source: "test-source", + SourceName: "test-source", Format: "pdf", ImagesFormat: "png", IssuesRange: "1-2", @@ -44,10 +44,10 @@ func TestInitializeCollectionFiltersIssues(t *testing.T) { site := &stubSite{ issues: []string{"url-1", "url-2", "url-3"}, - comics: map[string]*core.Comic{ - "url-1": {Name: "series", IssueNumber: "issue-1", URLSource: "url-1"}, - "url-2": {Name: "series", IssueNumber: "issue-2", URLSource: "url-2"}, - "url-3": {Name: "series", IssueNumber: "issue-3", URLSource: "url-3"}, + comics: map[string]*core.ComicIssue{ + "url-1": {Name: "series", IssueNumber: "issue-1", Source: &core.ComicSource{Name: "test-source", URL: "url-1"}}, + "url-2": {Name: "series", IssueNumber: "issue-2", Source: &core.ComicSource{Name: "test-source", URL: "url-2"}}, + "url-3": {Name: "series", IssueNumber: "issue-3", Source: &core.ComicSource{Name: "test-source", URL: "url-3"}}, }, } @@ -59,7 +59,7 @@ func TestInitializeCollectionFiltersIssues(t *testing.T) { } func TestLoadComicFromSourceUnknown(t *testing.T) { - options := &config.Options{Source: "unknown"} + options := &config.Options{SourceName: "unknown"} collection, err := LoadComicFromSource(options) require.Error(t, err) require.Empty(t, collection) diff --git a/pkg/sites/mangadex.go b/pkg/sites/mangadex.go index e7e3b942..560e8b65 100644 --- a/pkg/sites/mangadex.go +++ b/pkg/sites/mangadex.go @@ -251,8 +251,8 @@ func (m *Mangadex) GetInfo(urlValue string) (string, string) { } // Initialize loads links and metadata from mangadex. -func (m *Mangadex) Initialize(comic *core.Comic) error { - parts := util.TrimAndSplitURL(comic.URLSource) +func (m *Mangadex) Initialize(comic *core.ComicIssue) error { + parts := util.TrimAndSplitURL(comic.Source.URL) if len(parts) < 5 { return fmt.Errorf("URL not supported") } diff --git a/pkg/sites/mangadex_test.go b/pkg/sites/mangadex_test.go index 92353413..26c1b22f 100644 --- a/pkg/sites/mangadex_test.go +++ b/pkg/sites/mangadex_test.go @@ -71,11 +71,11 @@ func newTestMangadex(t *testing.T) (*Mangadex, func()) { ) opts := &config.Options{ - URL: server.URL + "/title/series-1/naruto", - Country: "en", - Source: "mangadex.org", - Logger: logger.NewLogger(false, nil), - Client: client, + URL: server.URL + "/title/series-1/naruto", + Country: "en", + SourceName: "mangadex.org", + Logger: logger.NewLogger(false, nil), + Client: client, } md := NewMangadex(opts) @@ -105,7 +105,9 @@ func TestMangadexInitialize(t *testing.T) { md, cleanup := newTestMangadex(t) defer cleanup() - comic := &core.Comic{URLSource: md.chapterBase + "/chapter-1"} + comic := &core.ComicIssue{ + Source: &core.ComicSource{Name: "test-source", URL: md.chapterBase + "/chapter-1"}, + } err := md.Initialize(comic) require.NoError(t, err) require.Equal(t, []string{ diff --git a/pkg/sites/mangakakalot.go b/pkg/sites/mangakakalot.go index a8523fe8..616e5c0d 100644 --- a/pkg/sites/mangakakalot.go +++ b/pkg/sites/mangakakalot.go @@ -22,7 +22,7 @@ func (m *MangaKakalot) GetInfo(url string) (string, string) { } // Initialize loads links and metadata from mangakakalot -func (m *MangaKakalot) Initialize(comic *core.Comic) error { +func (m *MangaKakalot) Initialize(comic *core.ComicIssue) error { return MangaKakalotInitialize(comic) } diff --git a/pkg/sites/mangakakalot_test.go b/pkg/sites/mangakakalot_test.go index 351709ee..714917d8 100644 --- a/pkg/sites/mangakakalot_test.go +++ b/pkg/sites/mangakakalot_test.go @@ -69,9 +69,9 @@ func TestMangaKakalotScraper(t *testing.T) { defer server.Close() opts := &config.Options{ - URL: server.URL + mangaKakalotListPath, - Source: "mangakakalot.com", - Logger: logger.NewLogger(false, nil), + URL: server.URL + mangaKakalotListPath, + SourceName: "mangakakalot.com", + Logger: logger.NewLogger(false, nil), } scraper := NewMangaKakalot(opts) @@ -79,7 +79,9 @@ func TestMangaKakalotScraper(t *testing.T) { require.Equal(t, "My Manga", title) require.Equal(t, "2", issue) - comic := &core.Comic{URLSource: server.URL + mangaKakalotChapterPath} + comic := &core.ComicIssue{ + Source: &core.ComicSource{Name: "test-source", URL: server.URL + mangaKakalotChapterPath}, + } require.NoError(t, scraper.Initialize(comic)) require.Equal(t, []string{ "https://cdn.example.com/manga-title/001.jpg", diff --git a/pkg/sites/manganato.go b/pkg/sites/manganato.go index 71689661..08cb1233 100644 --- a/pkg/sites/manganato.go +++ b/pkg/sites/manganato.go @@ -22,7 +22,7 @@ func (m *Manganato) GetInfo(url string) (string, string) { } // Initialize loads links and metadata from manganato -func (m *Manganato) Initialize(comic *core.Comic) error { +func (m *Manganato) Initialize(comic *core.ComicIssue) error { return MangaKakalotInitialize(comic) } diff --git a/pkg/sites/manganato_test.go b/pkg/sites/manganato_test.go index 23a85a6d..f41ac59b 100644 --- a/pkg/sites/manganato_test.go +++ b/pkg/sites/manganato_test.go @@ -66,9 +66,9 @@ func TestManganatoScraper(t *testing.T) { defer server.Close() opts := &config.Options{ - URL: server.URL + manganatoListPath, - Source: "manganato.com", - Logger: logger.NewLogger(false, nil), + URL: server.URL + manganatoListPath, + SourceName: "manganato.com", + Logger: logger.NewLogger(false, nil), } scraper := NewManganato(opts) @@ -77,7 +77,9 @@ func TestManganatoScraper(t *testing.T) { require.Equal(t, "My Manga", title) require.Equal(t, "2", issue) - comic := &core.Comic{URLSource: server.URL + manganatoChapterPath} + comic := &core.ComicIssue{ + Source: &core.ComicSource{Name: "test-source", URL: server.URL + manganatoChapterPath}, + } require.NoError(t, scraper.Initialize(comic)) require.Equal(t, []string{ "https://cdn.example.com/manga-title/001.jpg", diff --git a/pkg/sites/mangareader.go b/pkg/sites/mangareader.go index 6f21e3fd..867e0423 100644 --- a/pkg/sites/mangareader.go +++ b/pkg/sites/mangareader.go @@ -22,10 +22,10 @@ func NewMangareader(options *config.Options) *Mangareader { } } -func (m *Mangareader) retrieveImageLinks(comic *core.Comic) ([]string, error) { +func (m *Mangareader) retrieveImageLinks(comic *core.ComicIssue) ([]string, error) { var links []string - response, err := soup.Get(comic.URLSource) + response, err := soup.Get(comic.Source.URL) if err != nil { return nil, err @@ -116,8 +116,8 @@ func (m *Mangareader) GetInfo(url string) (string, string) { } // Initialize loads links and metadata from mangareader -func (m *Mangareader) Initialize(comic *core.Comic) error { - name, issueNumber := m.GetInfo(comic.URLSource) +func (m *Mangareader) Initialize(comic *core.ComicIssue) error { + name, issueNumber := m.GetInfo(comic.Source.URL) comic.Name = name comic.IssueNumber = issueNumber diff --git a/pkg/sites/mangareader_test.go b/pkg/sites/mangareader_test.go index 9b521c9c..e4af88be 100644 --- a/pkg/sites/mangareader_test.go +++ b/pkg/sites/mangareader_test.go @@ -66,7 +66,9 @@ func TestMangareaderScraper(t *testing.T) { scraper := NewMangareader(opts) - comic := &core.Comic{URLSource: server.URL + mangareaderIssuePath} + comic := &core.ComicIssue{ + Source: &core.ComicSource{Name: "test-source", URL: server.URL + mangareaderIssuePath}, + } require.NoError(t, scraper.Initialize(comic)) require.Equal(t, []string{ "https://cdn.example.com/naruto/001.jpg", diff --git a/pkg/sites/mangatown.go b/pkg/sites/mangatown.go index 54776d26..33b88972 100644 --- a/pkg/sites/mangatown.go +++ b/pkg/sites/mangatown.go @@ -36,11 +36,11 @@ func (m *Mangatown) findPages(document *soup.Root) []string { return pages } -func (m *Mangatown) retrieveImageLinks(comic *core.Comic) ([]string, error) { +func (m *Mangatown) retrieveImageLinks(comic *core.ComicIssue) ([]string, error) { var links []string var link string - response, err := soup.Get(comic.URLSource) + response, err := soup.Get(comic.Source.URL) if err != nil { return nil, err @@ -50,7 +50,7 @@ func (m *Mangatown) retrieveImageLinks(comic *core.Comic) ([]string, error) { pages := m.findPages(&document) for _, page := range pages { - link = fmt.Sprintf("%s%s.html", comic.URLSource, page) + link = fmt.Sprintf("%s%s.html", comic.Source.URL, page) response, err := soup.Get(link) if err != nil { @@ -139,7 +139,7 @@ func (m *Mangatown) GetInfo(url string) (string, string) { } // Initialize loads links and metadata from mangatown -func (m *Mangatown) Initialize(comic *core.Comic) error { +func (m *Mangatown) Initialize(comic *core.ComicIssue) error { links, err := m.retrieveImageLinks(comic) comic.Links = links diff --git a/pkg/sites/mangatown_test.go b/pkg/sites/mangatown_test.go index 79f4815e..a87e4cc6 100644 --- a/pkg/sites/mangatown_test.go +++ b/pkg/sites/mangatown_test.go @@ -79,7 +79,9 @@ func TestMangatownScraper(t *testing.T) { scraper := NewMangatown(opts) - comic := &core.Comic{URLSource: server.URL + mangatownIssuePath} + comic := &core.ComicIssue{ + Source: &core.ComicSource{Name: "test-source", URL: server.URL + mangatownIssuePath}, + } require.NoError(t, scraper.Initialize(comic)) require.Equal(t, []string{ "https://cdn.example.com/naruto/001.jpg", diff --git a/pkg/sites/readallcomics.go b/pkg/sites/readallcomics.go index c1795284..0eee3463 100644 --- a/pkg/sites/readallcomics.go +++ b/pkg/sites/readallcomics.go @@ -24,10 +24,10 @@ func NewReadallcomics(options *config.Options) *Readallcomics { } } -func (r *Readallcomics) retrieveImageLinks(comic *core.Comic) ([]string, error) { +func (r *Readallcomics) retrieveImageLinks(comic *core.ComicIssue) ([]string, error) { var links []string - response, err := soup.Get(comic.URLSource) + response, err := soup.Get(comic.Source.URL) if err != nil { return links, err } @@ -403,7 +403,7 @@ func isNumeric(s string) bool { } // Initialize prepare the comic instance with links and images. -func (r *Readallcomics) Initialize(comic *core.Comic) error { +func (r *Readallcomics) Initialize(comic *core.ComicIssue) error { links, err := r.retrieveImageLinks(comic) comic.Links = links diff --git a/pkg/sites/readallcomics_test.go b/pkg/sites/readallcomics_test.go index 340eb795..eb520a4d 100644 --- a/pkg/sites/readallcomics_test.go +++ b/pkg/sites/readallcomics_test.go @@ -62,7 +62,9 @@ func TestReadAllComicsScraper(t *testing.T) { scraper := NewReadallcomics(opts) - comic := &core.Comic{URLSource: server.URL + readAllIssuePath} + comic := &core.ComicIssue{ + Source: &core.ComicSource{Name: "test-source", URL: server.URL + readAllIssuePath}, + } require.NoError(t, scraper.Initialize(comic)) require.Equal(t, []string{ "https://cdn.example.com/sandman/001.jpg", diff --git a/pkg/sites/readcomiconline.go b/pkg/sites/readcomiconline.go index 8a295979..2d00f30f 100644 --- a/pkg/sites/readcomiconline.go +++ b/pkg/sites/readcomiconline.go @@ -61,12 +61,12 @@ func deobfuscateUrl(imageLink string) (string, error) { return link, nil } -func (c *ReadComicOnline) retrieveImageLinks(comic *core.Comic) ([]string, error) { +func (c *ReadComicOnline) retrieveImageLinks(comic *core.ComicIssue) ([]string, error) { var links []string const debugSnippetLimit = 4096 - comic.URLSource = strings.Split(comic.URLSource, "?")[0] - fetchURL := comic.URLSource + "?quality=hd&readType=1" + comic.Source.URL = strings.Split(comic.Source.URL, "?")[0] + fetchURL := comic.Source.URL + "?quality=hd&readType=1" if c.options.Debug && c.options.Logger != nil { c.options.Logger.Debugf("readcomiconline: fetching %s", fetchURL) @@ -97,7 +97,7 @@ func (c *ReadComicOnline) retrieveImageLinks(comic *core.Comic) ([]string, error } if c.options.Debug && c.options.Logger != nil { - c.options.Logger.Debugf("readcomiconline: found %d obfuscated entries, %d valid links for %s", len(match), len(links), comic.URLSource) + c.options.Logger.Debugf("readcomiconline: found %d obfuscated entries, %d valid links for %s", len(match), len(links), comic.Source.URL) snippet := response if len(snippet) > debugSnippetLimit { snippet = snippet[:debugSnippetLimit] @@ -199,7 +199,7 @@ func (c *ReadComicOnline) GetInfo(url string) (string, string) { // Initialize will initialize the comic based // on ReadComicOnline.to -func (c *ReadComicOnline) Initialize(comic *core.Comic) error { +func (c *ReadComicOnline) Initialize(comic *core.ComicIssue) error { links, err := c.retrieveImageLinks(comic) comic.Links = links diff --git a/pkg/sites/readcomiconline_test.go b/pkg/sites/readcomiconline_test.go index b7dbaa57..ab7fa86c 100644 --- a/pkg/sites/readcomiconline_test.go +++ b/pkg/sites/readcomiconline_test.go @@ -61,7 +61,9 @@ func TestReadComicOnlineScraper(t *testing.T) { scraper := NewReadComiconline(opts) - comic := &core.Comic{URLSource: server.URL + rcoIssuePath} + comic := &core.ComicIssue{ + Source: &core.ComicSource{Name: "test-source", URL: server.URL + rcoIssuePath}, + } require.NoError(t, scraper.Initialize(comic)) require.Equal(t, []string{ "https://2.bp.blogspot.com/abc123=s1600?", From 0b9a272de2e370e4ac68fd2baca542b8a7d5a5f6 Mon Sep 17 00:00:00 2001 From: ProjectDislocate <173194962+ProjectDislocate@users.noreply.github.com> Date: Fri, 6 Mar 2026 08:57:36 -0500 Subject: [PATCH 19/79] feat: validate output format --- cmd/downloader/main.go | 6 ++-- cmd/downloader/main_test.go | 8 ++--- cmd/gui/gui.go | 2 +- go.mod | 1 - go.sum | 7 ----- pkg/config/options.go | 2 +- pkg/core/core.go | 49 +++++++++++++------------------ pkg/core/core_test.go | 14 ++++----- pkg/core/output.go | 34 +++++++++++++++++++++ pkg/core/output_test.go | 23 +++++++++++++++ pkg/sites/comicextra.go | 2 +- pkg/sites/comicextra_test.go | 2 +- pkg/sites/common.go | 2 +- pkg/sites/loader.go | 12 ++++++-- pkg/sites/loader_test.go | 2 +- pkg/sites/mangadex.go | 2 +- pkg/sites/mangadex_test.go | 2 +- pkg/sites/mangakakalot_test.go | 2 +- pkg/sites/manganato_test.go | 2 +- pkg/sites/mangareader.go | 2 +- pkg/sites/mangareader_test.go | 2 +- pkg/sites/mangatown.go | 2 +- pkg/sites/mangatown_test.go | 2 +- pkg/sites/readallcomics.go | 2 +- pkg/sites/readallcomics_test.go | 2 +- pkg/sites/readcomiconline.go | 2 +- pkg/sites/readcomiconline_test.go | 2 +- 27 files changed, 119 insertions(+), 71 deletions(-) create mode 100644 pkg/core/output.go create mode 100644 pkg/core/output_test.go diff --git a/cmd/downloader/main.go b/cmd/downloader/main.go index 80232d5d..55e7fdad 100644 --- a/cmd/downloader/main.go +++ b/cmd/downloader/main.go @@ -25,7 +25,7 @@ var ( country string // manga/comic final output forceAspect bool - format string + outputFormat string customComicName string // force only issue number filenames issueNumberNameOnly bool @@ -62,7 +62,7 @@ func init() { flag.BoolVar(&createDefaultPath, "create-default-path", true, "Using this flag your comics/issue will be downloaded without prepending the default folder structure, `comics/[source]/[name]/`") flag.StringVar(&country, "country", "", "Set the country to retrieve a manga, Used by MangaDex which uses ISO 3166-1 codes") flag.BoolVar(&forceAspect, "force-aspect", false, "Force images to A4 Portrait aspect ratio") - flag.StringVar(&format, "format", "pdf", "Comic format output, supported formats are pdf,epub,cbr,cbz") + flag.StringVar(&outputFormat, "format", "pdf", "Comic format output, supported formats are pdf,epub,cbr,cbz") flag.StringVar(&customComicName, "custom-comic-name", "", "Use a custom name for the comic output.") flag.StringVar(&imagesFormat, "images-format", "jpg", "To use with `images-only` flag, choose the image format, available png,jpeg,img") flag.BoolVar(&issueNumberNameOnly, "issue-number-only", false, "Force only saving with issue number instead of chapter name + issue number.") @@ -89,7 +89,7 @@ func buildOptions() config.Options { IssueNumberNameOnly: issueNumberNameOnly, URL: url, ForceAspect: forceAspect, - Format: format, + OutputFormat: outputFormat, CustomComicName: customComicName, Daemon: daemon, DaemonTimeout: daemonTimeout, diff --git a/cmd/downloader/main_test.go b/cmd/downloader/main_test.go index 2a1d50ab..ae5744d9 100644 --- a/cmd/downloader/main_test.go +++ b/cmd/downloader/main_test.go @@ -33,7 +33,7 @@ func TestBuildOptionsCopiesGlobals(t *testing.T) { imagesFormat: imagesFormat, country: country, forceAspect: forceAspect, - format: format, + format: outputFormat, customComicName: customComicName, issueNumberNameOnly: issueNumberNameOnly, url: url, @@ -54,7 +54,7 @@ func TestBuildOptionsCopiesGlobals(t *testing.T) { imagesFormat = prev.imagesFormat country = prev.country forceAspect = prev.forceAspect - format = prev.format + outputFormat = prev.format customComicName = prev.customComicName issueNumberNameOnly = prev.issueNumberNameOnly url = prev.url @@ -75,7 +75,7 @@ func TestBuildOptionsCopiesGlobals(t *testing.T) { imagesFormat = "png" country = "jp" forceAspect = true - format = "epub" + outputFormat = "epub" customComicName = "custom" issueNumberNameOnly = true url = "http://example.com/comic" @@ -94,7 +94,7 @@ func TestBuildOptionsCopiesGlobals(t *testing.T) { t.Fatalf("expected boolean flags to be copied into options: %+v", opts) } - if opts.ImagesFormat != "png" || opts.Country != "jp" || opts.Format != "epub" { + if opts.ImagesFormat != "png" || opts.Country != "jp" || opts.OutputFormat != "epub" { t.Fatalf("expected string values to be copied, got %+v", opts) } diff --git a/cmd/gui/gui.go b/cmd/gui/gui.go index c3c44873..64f80372 100644 --- a/cmd/gui/gui.go +++ b/cmd/gui/gui.go @@ -47,7 +47,7 @@ func (d *Downloader) Submit() { All: d.AllChapters.Checked, Last: d.LastChapter.Checked, URL: strings.TrimSpace(d.URL.Text), - Format: d.Format.Selected, + OutputFormat: d.Format.Selected, Country: d.Country.Text, ImagesFormat: d.ImagesFormat.Selected, ImagesOnly: d.ImagesOnly.Checked, diff --git a/go.mod b/go.mod index 4b114139..628733e5 100644 --- a/go.mod +++ b/go.mod @@ -35,7 +35,6 @@ require ( github.com/godbus/dbus/v5 v5.0.3 // indirect github.com/gofrs/uuid v3.1.0+incompatible // indirect github.com/goki/freetype v0.0.0-20181231101311-fa8a33aabaff // indirect - github.com/google/go-cmp v0.5.9 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect diff --git a/go.sum b/go.sum index e97de3d6..c40e8ddc 100644 --- a/go.sum +++ b/go.sum @@ -88,14 +88,7 @@ github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5a github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= -github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY= -github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= -github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= -github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= diff --git a/pkg/config/options.go b/pkg/config/options.go index e40a6e5f..a46846fd 100644 --- a/pkg/config/options.go +++ b/pkg/config/options.go @@ -24,7 +24,7 @@ type Options struct { DaemonTimeout int ImagesFormat string Country string - Format string + OutputFormat string CustomComicName string ForceAspect bool OutputFolder string diff --git a/pkg/core/core.go b/pkg/core/core.go index a05152e5..61f36e3c 100644 --- a/pkg/core/core.go +++ b/pkg/core/core.go @@ -33,26 +33,19 @@ import ( // DefaultMessage for correctly saved file const DefaultMessage = "file correctly saved" -// manga output format supported -const ( - CBR = "cbr" - CBZ = "cbz" - EPUB = "epub" - PDF = "pdf" -) - type ComicSource struct { Name string - URL string + URL string // URL of the comic/manga issue } // ComicIssue struct contains all the informations about a comic type ComicIssue struct { - Author string - Name string - IssueNumber string - Links []string - Format string + Author string + Name string + IssueNumber string + + ImageLinks []string + OutputFormat ComicOutputFormat ImagesFormat string Source *ComicSource @@ -61,7 +54,7 @@ type ComicIssue struct { // DownloadResult captures the outcome of downloading a comic's images. type DownloadResult struct { Dir string - FilePaths []string + FilePaths []string // Absolute paths to the downloaded image files } func ensureClient(options *config.Options) *httpclient.ComicClient { @@ -103,12 +96,12 @@ func (comic *ComicIssue) makeEPUB(options *config.Options, images *DownloadResul return err } - if err = e.Write(util.GetPathToFile(dir, comic.Name, comic.IssueNumber, comic.Format, options.IssueNumberNameOnly)); err != nil { + if err = e.Write(util.GetPathToFile(dir, comic.Name, comic.IssueNumber, comic.OutputFormat.String(), options.IssueNumberNameOnly)); err != nil { return err } if options.Logger != nil { - options.Logger.Infof("%s %s", strings.ToUpper(comic.Format), DefaultMessage) + options.Logger.Infof("%s %s", strings.ToUpper(comic.OutputFormat.String()), DefaultMessage) } return nil } @@ -162,13 +155,13 @@ func (comic *ComicIssue) makePDF(options *config.Options, images *DownloadResult return err } - filePath := util.GetPathToFile(dir, comic.Name, comic.IssueNumber, comic.Format, options.IssueNumberNameOnly) + filePath := util.GetPathToFile(dir, comic.Name, comic.IssueNumber, comic.OutputFormat.String(), options.IssueNumberNameOnly) if err = pdf.OutputFileAndClose(filePath); err != nil { return err } if options.Logger != nil { - options.Logger.Infof("%s %s", strings.ToUpper(comic.Format), DefaultMessage) + options.Logger.Infof("%s %s", strings.ToUpper(comic.OutputFormat.String()), DefaultMessage) } return nil } @@ -181,7 +174,7 @@ func (comic *ComicIssue) makeCBRZ(options *config.Options, images *DownloadResul } zipArchiveName := filepath.Join(dir, fmt.Sprintf("%s.zip", comic.IssueNumber)) - newName := util.GetPathToFile(dir, comic.Name, comic.IssueNumber, comic.Format, options.IssueNumberNameOnly) + newName := util.GetPathToFile(dir, comic.Name, comic.IssueNumber, comic.OutputFormat.String(), options.IssueNumberNameOnly) out, err := os.Create(zipArchiveName) if err != nil { @@ -220,14 +213,14 @@ func (comic *ComicIssue) makeCBRZ(options *config.Options, images *DownloadResul } if options.Logger != nil { - options.Logger.Infof("%s %s", strings.ToUpper(comic.Format), DefaultMessage) + options.Logger.Infof("%s %s", strings.ToUpper(comic.OutputFormat.String()), DefaultMessage) } return nil } // DownloadImages will download the comic/manga images. func (comic *ComicIssue) DownloadImages(options *config.Options) (*DownloadResult, error) { - if len(comic.Links) == 0 { + if len(comic.ImageLinks) == 0 { return nil, fmt.Errorf("download failed, no links found for: %s", comic.Source.URL) } @@ -239,7 +232,7 @@ func (comic *ComicIssue) DownloadImages(options *config.Options) (*DownloadResul } existing, err := readExistingImages(dir) - if err == nil && len(existing) == len(comic.Links) && len(existing) > 0 { + if err == nil && len(existing) == len(comic.ImageLinks) && len(existing) > 0 { return &DownloadResult{Dir: dir, FilePaths: existing}, nil } @@ -250,7 +243,7 @@ func (comic *ComicIssue) DownloadImages(options *config.Options) (*DownloadResul return nil, err } - progress := progressbar.NewOptions(len(comic.Links), progressbar.OptionSetRenderBlankState(true)) + progress := progressbar.NewOptions(len(comic.ImageLinks), progressbar.OptionSetRenderBlankState(true)) format := util.ImageType(comic.ImagesFormat) requestDelay := options.RequestDelay @@ -271,15 +264,15 @@ func (comic *ComicIssue) DownloadImages(options *config.Options) (*DownloadResul link string } - jobs := make([]downloadJob, 0, len(comic.Links)) - for idx, link := range comic.Links { + jobs := make([]downloadJob, 0, len(comic.ImageLinks)) + for idx, link := range comic.ImageLinks { if strings.TrimSpace(link) == "" { continue } jobs = append(jobs, downloadJob{index: idx, link: link}) } - results := make([]string, len(comic.Links)) + results := make([]string, len(comic.ImageLinks)) ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) defer cancel() @@ -459,7 +452,7 @@ func (comic *ComicIssue) MakeComic(options *config.Options) error { } }() - switch comic.Format { + switch comic.OutputFormat { case EPUB: return comic.makeEPUB(options, result) case CBR, CBZ: diff --git a/pkg/core/core_test.go b/pkg/core/core_test.go index e729fb92..5c738b2e 100644 --- a/pkg/core/core_test.go +++ b/pkg/core/core_test.go @@ -72,7 +72,7 @@ func TestDownloadImagesCreatesFiles(t *testing.T) { Source: &ComicSource{Name: "test-source", URL: server.URL}, IssueNumber: "1", ImagesFormat: "png", - Links: buildLinks(server, 3), + ImageLinks: buildLinks(server, 3), } result, err := comic.DownloadImages(opts) @@ -99,9 +99,9 @@ func TestMakeComicPDF(t *testing.T) { Name: "foo", Source: &ComicSource{Name: "test-source", URL: server.URL}, IssueNumber: "1", - Format: PDF, + OutputFormat: PDF, ImagesFormat: "png", - Links: buildLinks(server, 2), + ImageLinks: buildLinks(server, 2), } require.NoError(t, comic.MakeComic(opts)) @@ -121,9 +121,9 @@ func TestMakeComicEPUB(t *testing.T) { Source: &ComicSource{Name: "test-source", URL: server.URL}, IssueNumber: "42", Author: "Author", - Format: EPUB, + OutputFormat: EPUB, ImagesFormat: "png", - Links: buildLinks(server, 2), + ImageLinks: buildLinks(server, 2), } require.NoError(t, comic.MakeComic(opts)) @@ -142,9 +142,9 @@ func TestMakeComicCBZ(t *testing.T) { Name: "baz", Source: &ComicSource{Name: "test-source", URL: server.URL}, IssueNumber: "7", - Format: CBZ, + OutputFormat: CBZ, ImagesFormat: "png", - Links: buildLinks(server, 2), + ImageLinks: buildLinks(server, 2), } require.NoError(t, comic.MakeComic(opts)) diff --git a/pkg/core/output.go b/pkg/core/output.go new file mode 100644 index 00000000..846d87b2 --- /dev/null +++ b/pkg/core/output.go @@ -0,0 +1,34 @@ +package core + +import "errors" + +type ComicOutputFormat string + +func (c ComicOutputFormat) String() string { + return string(c) +} + +// manga output format supported +const ( + CBR ComicOutputFormat = "cbr" + CBZ ComicOutputFormat = "cbz" + EPUB ComicOutputFormat = "epub" + PDF ComicOutputFormat = "pdf" +) + +var InvalidOutputFormatError = errors.New("Invalid output format") + +func ToComicOutputFormat(format string) (ComicOutputFormat, error) { + switch format { + case "cbr": + return CBR, nil + case "cbz": + return CBZ, nil + case "epub": + return EPUB, nil + case "pdf": + return PDF, nil + default: + return "", InvalidOutputFormatError + } +} diff --git a/pkg/core/output_test.go b/pkg/core/output_test.go new file mode 100644 index 00000000..bc3737ff --- /dev/null +++ b/pkg/core/output_test.go @@ -0,0 +1,23 @@ +package core + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestComicOutputFormatHandlesInvalid(t *testing.T) { + _, err := ToComicOutputFormat("invalid_format") + + if assert.Error(t, err) { + assert.Equal(t, InvalidOutputFormatError, err) + } +} + +func TestComicOutputFormatHandlesValid(t *testing.T) { + format, err := ToComicOutputFormat("cbz") + + if assert.NoError(t, err) { + assert.Equal(t, CBZ, format) + } +} diff --git a/pkg/sites/comicextra.go b/pkg/sites/comicextra.go index 7f514d19..7f80cb75 100644 --- a/pkg/sites/comicextra.go +++ b/pkg/sites/comicextra.go @@ -162,7 +162,7 @@ func (c *Comicextra) GetInfo(url string) (string, string) { // on comicextra.com func (c *Comicextra) Initialize(comic *core.ComicIssue) error { links, err := c.retrieveImageLinks(comic) - comic.Links = links + comic.ImageLinks = links return err } diff --git a/pkg/sites/comicextra_test.go b/pkg/sites/comicextra_test.go index f7fde77c..830e7d5e 100644 --- a/pkg/sites/comicextra_test.go +++ b/pkg/sites/comicextra_test.go @@ -79,7 +79,7 @@ func TestComicExtraScraper(t *testing.T) { require.Equal(t, []string{ "https://cdn.example.com/batman?page=1", "https://cdn.example.com/batman?page=2", - }, comic.Links) + }, comic.ImageLinks) } func TestComicExtraRetrieveIssueLinksAll(t *testing.T) { diff --git a/pkg/sites/common.go b/pkg/sites/common.go index ba8caa42..945270bf 100644 --- a/pkg/sites/common.go +++ b/pkg/sites/common.go @@ -51,7 +51,7 @@ func MangaKakalotInitialize(comic *core.ComicIssue) error { for _, img := range f.FindAll("img") { links = append(links, img.Attrs()["src"]) } - comic.Links = links + comic.ImageLinks = links return nil } diff --git a/pkg/sites/loader.go b/pkg/sites/loader.go index a7a46427..394956ab 100644 --- a/pkg/sites/loader.go +++ b/pkg/sites/loader.go @@ -14,7 +14,7 @@ import ( func initializeCollection(issues []string, options *config.Options, base BaseSite) ([]*core.ComicIssue, error) { var collection []*core.ComicIssue - var err error + // var err error if len(issues) == 0 { return collection, fmt.Errorf("no issues found for URL %q; ensure it points to a specific comic or chapter page", options.URL) @@ -42,13 +42,19 @@ func initializeCollection(issues []string, options *config.Options, base BaseSit continue } + outputFormat, err := core.ToComicOutputFormat(options.OutputFormat) + if err != nil { + return collection, err + } + dir, pathErr := util.PathSetup(options.CreateDefaultPath, options.OutputFolder, options.SourceName, name) if pathErr != nil { return collection, pathErr } - fileName := util.GetPathToFile(dir, name, issueNumber, options.Format, options.IssueNumberNameOnly) + fileName := util.GetPathToFile(dir, name, issueNumber, outputFormat.String(), options.IssueNumberNameOnly) if util.DirectoryOrFileDoesNotExist(fileName) || options.ImagesOnly { + comic := &core.ComicIssue{ Name: name, IssueNumber: issueNumber, @@ -56,7 +62,7 @@ func initializeCollection(issues []string, options *config.Options, base BaseSit Name: options.SourceName, URL: url, }, - Format: options.Format, + OutputFormat: outputFormat, ImagesFormat: options.ImagesFormat, } if err = base.Initialize(comic); err != nil { diff --git a/pkg/sites/loader_test.go b/pkg/sites/loader_test.go index 833c04dc..a8d90be9 100644 --- a/pkg/sites/loader_test.go +++ b/pkg/sites/loader_test.go @@ -36,7 +36,7 @@ func (s *stubSite) RetrieveIssueLinks() ([]string, error) { func TestInitializeCollectionFiltersIssues(t *testing.T) { options := &config.Options{ SourceName: "test-source", - Format: "pdf", + OutputFormat: "pdf", ImagesFormat: "png", IssuesRange: "1-2", All: true, diff --git a/pkg/sites/mangadex.go b/pkg/sites/mangadex.go index 560e8b65..f099e13b 100644 --- a/pkg/sites/mangadex.go +++ b/pkg/sites/mangadex.go @@ -260,6 +260,6 @@ func (m *Mangadex) Initialize(comic *core.ComicIssue) error { if err != nil { return err } - comic.Links = images + comic.ImageLinks = images return nil } diff --git a/pkg/sites/mangadex_test.go b/pkg/sites/mangadex_test.go index 26c1b22f..2060ed3b 100644 --- a/pkg/sites/mangadex_test.go +++ b/pkg/sites/mangadex_test.go @@ -113,7 +113,7 @@ func TestMangadexInitialize(t *testing.T) { require.Equal(t, []string{ md.uploadsBase + "/HASH/001.png", md.uploadsBase + "/HASH/002.png", - }, comic.Links) + }, comic.ImageLinks) } func TestMangadexGetInfo(t *testing.T) { diff --git a/pkg/sites/mangakakalot_test.go b/pkg/sites/mangakakalot_test.go index 714917d8..ac8eabeb 100644 --- a/pkg/sites/mangakakalot_test.go +++ b/pkg/sites/mangakakalot_test.go @@ -86,7 +86,7 @@ func TestMangaKakalotScraper(t *testing.T) { require.Equal(t, []string{ "https://cdn.example.com/manga-title/001.jpg", "https://cdn.example.com/manga-title/002.jpg", - }, comic.Links) + }, comic.ImageLinks) links, err := scraper.RetrieveIssueLinks() require.NoError(t, err) diff --git a/pkg/sites/manganato_test.go b/pkg/sites/manganato_test.go index f41ac59b..3719efb0 100644 --- a/pkg/sites/manganato_test.go +++ b/pkg/sites/manganato_test.go @@ -84,7 +84,7 @@ func TestManganatoScraper(t *testing.T) { require.Equal(t, []string{ "https://cdn.example.com/manga-title/001.jpg", "https://cdn.example.com/manga-title/002.jpg", - }, comic.Links) + }, comic.ImageLinks) links, err := scraper.RetrieveIssueLinks() require.NoError(t, err) diff --git a/pkg/sites/mangareader.go b/pkg/sites/mangareader.go index 867e0423..32769498 100644 --- a/pkg/sites/mangareader.go +++ b/pkg/sites/mangareader.go @@ -122,7 +122,7 @@ func (m *Mangareader) Initialize(comic *core.ComicIssue) error { comic.IssueNumber = issueNumber links, err := m.retrieveImageLinks(comic) - comic.Links = links + comic.ImageLinks = links return err } diff --git a/pkg/sites/mangareader_test.go b/pkg/sites/mangareader_test.go index e4af88be..75af022c 100644 --- a/pkg/sites/mangareader_test.go +++ b/pkg/sites/mangareader_test.go @@ -73,7 +73,7 @@ func TestMangareaderScraper(t *testing.T) { require.Equal(t, []string{ "https://cdn.example.com/naruto/001.jpg", "https://cdn.example.com/naruto/002.jpg", - }, comic.Links) + }, comic.ImageLinks) opts.All = true opts.URL = server.URL + mangareaderIssuePath diff --git a/pkg/sites/mangatown.go b/pkg/sites/mangatown.go index 33b88972..b9b40cc3 100644 --- a/pkg/sites/mangatown.go +++ b/pkg/sites/mangatown.go @@ -141,7 +141,7 @@ func (m *Mangatown) GetInfo(url string) (string, string) { // Initialize loads links and metadata from mangatown func (m *Mangatown) Initialize(comic *core.ComicIssue) error { links, err := m.retrieveImageLinks(comic) - comic.Links = links + comic.ImageLinks = links return err } diff --git a/pkg/sites/mangatown_test.go b/pkg/sites/mangatown_test.go index a87e4cc6..7d9772f6 100644 --- a/pkg/sites/mangatown_test.go +++ b/pkg/sites/mangatown_test.go @@ -86,7 +86,7 @@ func TestMangatownScraper(t *testing.T) { require.Equal(t, []string{ "https://cdn.example.com/naruto/001.jpg", "https://cdn.example.com/naruto/002.jpg", - }, comic.Links) + }, comic.ImageLinks) opts.All = true scraper = NewMangatown(opts) diff --git a/pkg/sites/readallcomics.go b/pkg/sites/readallcomics.go index 0eee3463..096d2e29 100644 --- a/pkg/sites/readallcomics.go +++ b/pkg/sites/readallcomics.go @@ -405,7 +405,7 @@ func isNumeric(s string) bool { // Initialize prepare the comic instance with links and images. func (r *Readallcomics) Initialize(comic *core.ComicIssue) error { links, err := r.retrieveImageLinks(comic) - comic.Links = links + comic.ImageLinks = links return err } diff --git a/pkg/sites/readallcomics_test.go b/pkg/sites/readallcomics_test.go index eb520a4d..39174636 100644 --- a/pkg/sites/readallcomics_test.go +++ b/pkg/sites/readallcomics_test.go @@ -69,7 +69,7 @@ func TestReadAllComicsScraper(t *testing.T) { require.Equal(t, []string{ "https://cdn.example.com/sandman/001.jpg", "https://cdn.example.com/sandman/002.jpg", - }, comic.Links) + }, comic.ImageLinks) // Category listing for All opts.All = true diff --git a/pkg/sites/readcomiconline.go b/pkg/sites/readcomiconline.go index 2d00f30f..484473f1 100644 --- a/pkg/sites/readcomiconline.go +++ b/pkg/sites/readcomiconline.go @@ -201,7 +201,7 @@ func (c *ReadComicOnline) GetInfo(url string) (string, string) { // on ReadComicOnline.to func (c *ReadComicOnline) Initialize(comic *core.ComicIssue) error { links, err := c.retrieveImageLinks(comic) - comic.Links = links + comic.ImageLinks = links return err } diff --git a/pkg/sites/readcomiconline_test.go b/pkg/sites/readcomiconline_test.go index ab7fa86c..bbdec83f 100644 --- a/pkg/sites/readcomiconline_test.go +++ b/pkg/sites/readcomiconline_test.go @@ -68,7 +68,7 @@ func TestReadComicOnlineScraper(t *testing.T) { require.Equal(t, []string{ "https://2.bp.blogspot.com/abc123=s1600?", "https://2.bp.blogspot.com/def456=s1600?", - }, comic.Links) + }, comic.ImageLinks) opts.All = true opts.URL = server.URL + rcoListPath From 6cf972d59bb93d9d3c1d057de884181fe018bdb1 Mon Sep 17 00:00:00 2001 From: ProjectDislocate <173194962+ProjectDislocate@users.noreply.github.com> Date: Fri, 6 Mar 2026 14:06:52 -0500 Subject: [PATCH 20/79] feat: comicinfo.xml support --- go.mod | 1 + go.sum | 2 + pkg/core/core.go | 24 ++---- pkg/core/core_test.go | 9 ++- pkg/core/metadata.go | 109 ++++++++++++++++++++++++++ pkg/core/output.go | 178 +++++++++++++++++++++++++++++++++++++++++- pkg/sites/loader.go | 9 ++- 7 files changed, 309 insertions(+), 23 deletions(-) create mode 100644 pkg/core/metadata.go diff --git a/go.mod b/go.mod index 628733e5..7f3c6ec9 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ toolchain go1.24.4 require ( fyne.io/fyne v1.4.3 github.com/anaskhan96/soup v1.2.5 + github.com/beevik/etree v1.6.0 github.com/bmaupin/go-epub v1.1.0 github.com/dlclark/regexp2 v1.10.0 github.com/jung-kurt/gofpdf v1.16.2 diff --git a/go.sum b/go.sum index c40e8ddc..3738e72d 100644 --- a/go.sum +++ b/go.sum @@ -27,6 +27,8 @@ github.com/anaskhan96/soup v1.2.5 h1:V/FHiusdTrPrdF4iA1YkVxsOpdNcgvqT1hG+YtcZ5hM github.com/anaskhan96/soup v1.2.5/go.mod h1:6YnEp9A2yywlYdM4EgDz9NEHclocMepEtku7wg6Cq3s= github.com/andybalholm/brotli v1.1.2-0.20250424173009-453214e765f3 h1:8PmGpDEZl9yDpcdEr6Odf23feCxK3LNUNMxjXg41pZQ= github.com/andybalholm/brotli v1.1.2-0.20250424173009-453214e765f3/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA= +github.com/beevik/etree v1.6.0 h1:u8Kwy8pp9D9XeITj2Z0XtA5qqZEmtJtuXZRQi+j03eE= +github.com/beevik/etree v1.6.0/go.mod h1:bh4zJxiIr62SOf9pRzN7UUYaEDa9HEKafK25+sLc0Gc= github.com/bmaupin/go-epub v1.1.0 h1:XJyvvjchtUlbZ2P7eaEeB8EFw2NgVY5ycREFpmd6MKM= github.com/bmaupin/go-epub v1.1.0/go.mod h1:mBan+0WgVv5JbPNw1xfnfQoTRN9iPMKBshZwPOL0SY0= github.com/bodgit/plumbing v1.3.0 h1:pf9Itz1JOQgn7vEOE7v7nlEfBykYqvUYioC61TwWCFU= diff --git a/pkg/core/core.go b/pkg/core/core.go index 61f36e3c..8261ce17 100644 --- a/pkg/core/core.go +++ b/pkg/core/core.go @@ -33,24 +33,6 @@ import ( // DefaultMessage for correctly saved file const DefaultMessage = "file correctly saved" -type ComicSource struct { - Name string - URL string // URL of the comic/manga issue -} - -// ComicIssue struct contains all the informations about a comic -type ComicIssue struct { - Author string - Name string - IssueNumber string - - ImageLinks []string - OutputFormat ComicOutputFormat - ImagesFormat string - - Source *ComicSource -} - // DownloadResult captures the outcome of downloading a comic's images. type DownloadResult struct { Dir string @@ -173,6 +155,11 @@ func (comic *ComicIssue) makeCBRZ(options *config.Options, images *DownloadResul return err } + comicinfoXMLPath, err := comic.makeComicInfoXML(options, images) + if err != nil { + return err + } + zipArchiveName := filepath.Join(dir, fmt.Sprintf("%s.zip", comic.IssueNumber)) newName := util.GetPathToFile(dir, comic.Name, comic.IssueNumber, comic.OutputFormat.String(), options.IssueNumberNameOnly) @@ -192,6 +179,7 @@ func (comic *ComicIssue) makeCBRZ(options *config.Options, images *DownloadResul for _, filePath := range images.FilePaths { fileMap[filePath] = path.Base(filePath) } + fileMap[comicinfoXMLPath] = "ComicInfo.xml" archiveFiles, err := archives.FilesFromDisk(context.Background(), nil, fileMap) if err != nil { diff --git a/pkg/core/core_test.go b/pkg/core/core_test.go index 5c738b2e..aff1324b 100644 --- a/pkg/core/core_test.go +++ b/pkg/core/core_test.go @@ -139,12 +139,17 @@ func TestMakeComicCBZ(t *testing.T) { opts := newTestOptions(t, server) comic := &ComicIssue{ - Name: "baz", - Source: &ComicSource{Name: "test-source", URL: server.URL}, + Name: "baz", + IssueNumber: "7", OutputFormat: CBZ, ImagesFormat: "png", ImageLinks: buildLinks(server, 2), + + Source: &ComicSource{Name: "test-source", URL: server.URL}, + SeriesMetadata: &SeriesMetadata{ + Title: "Baz Series", + }, } require.NoError(t, comic.MakeComic(opts)) diff --git a/pkg/core/metadata.go b/pkg/core/metadata.go new file mode 100644 index 00000000..ee5176e6 --- /dev/null +++ b/pkg/core/metadata.go @@ -0,0 +1,109 @@ +package core + +import ( + "time" +) + +type AgeRating string + +// Based on https://anansi-project.github.io/docs/comicinfo/schemas/v2.1 +const ( + AgeRatingUnrated AgeRating = "Unrated" + AgeRatingAO18 AgeRating = "Adults Only 18+" + AgeRatingEarlyChildhood AgeRating = "Early Childhood" + AgeRatingEveryone AgeRating = "Everyone" + AgeRatingEveryone10 AgeRating = "Everyone 10+" + AgeRatingG AgeRating = "G" + AgeRatingKidsToAdults AgeRating = "Kids to Adults" + AgeRatingM AgeRating = "M" + AgeRatingMA15 AgeRating = "MA15+" + AgeRatingMature AgeRating = "Mature 17+" + AgeRatingPG AgeRating = "PG" + AgeRatingR18 AgeRating = "R18+" + AgeRatingRatingPending AgeRating = "Rating Pending" + AgeRatingTeen AgeRating = "Teen" + AgeRatingX18 AgeRating = "X18+" +) + +type ComicSource struct { + Name string + URL string // URL of the comic/manga issue +} + +type CreatorsRole string + +type SeriesCreator struct { + Name string + Role CreatorsRole +} + +type SeriesMetadata struct { + Title string // Series title, should be in the native language of the comic/manga when possible + LocalizedTitle map[string]string // Map of language code to localized title, e.g. {"en": "One Piece", "jp": "ワンピース"} + Description map[string]string // Map of language code to description, e.g. {"en": "A story about pirates...", "jp": "海賊の物語..."} + Creators []SeriesCreator + + IsManga *bool // True if it's a manga, false if it's a comic + IsRTL *bool // True if the comic/manga is read right-to-left, false if left-to-right. Only relevant for manga, but some comics may also be RTL. + + CommunityRating *float64 // Average rating from the community, 0-5 + AgeRating *AgeRating + Tags []string // ninja or school life + Genres []string // eg Science-Fiction or Shonen + + WebLinks []string // Official website, social media, etc. +} + +// ComicIssue struct contains all the informations about a comic +type ComicIssue struct { + Author string // Remove in favor of SeriesMetadata.Creators?? + Name string // Issue name/title + + IssueNumber string + Volume *string + LanguageISO *string // IETF language tag + ReleaseDate *time.Time + + ImageLinks []string + OutputFormat ComicOutputFormat + ImagesFormat string + + Source *ComicSource + SeriesMetadata *SeriesMetadata +} + +const ( + CreatorRoleUnknown = "Unknown" + CreatorRoleWriter = "Writer" + CreatorRolePenciller = "Penciller" + CreatorRoleInker = "Inker" + CreatorRoleColorist = "Colorist" + CreatorRoleLetterer = "Letterer" + CreatorRoleCoverArtist = "CoverArtist" + CreatorRoleEditor = "Editor" + CreatorRoleTranslator = "Translator" +) + +// func SourceAuthorRoleToSeriesAuthorRole(sourceRole string) string { +// sourceRole = strings.ToLower(strings.TrimSpace(sourceRole)) +// switch sourceRole { +// case "writer", "author": +// return CreatorRoleWriter +// case "penciller": +// return CreatorRolePenciller +// case "inker": +// return CreatorRoleInker +// case "colorist": +// return CreatorRoleColorist +// case "letterer": +// return CreatorRoleLetterer +// case "cover artist", "coverartist", "cover-artist": +// return CreatorRoleCoverArtist +// case "editor": +// return CreatorRoleEditor +// case "translator": +// return CreatorRoleTranslator +// default: +// return CreatorRoleUnknown +// } +// } diff --git a/pkg/core/output.go b/pkg/core/output.go index 846d87b2..72bd0a20 100644 --- a/pkg/core/output.go +++ b/pkg/core/output.go @@ -1,6 +1,19 @@ package core -import "errors" +import ( + "errors" + "fmt" + "net/url" + "os" + "path/filepath" + "strings" + "time" + + "github.com/Girbons/comics-downloader/internal/version" + "github.com/Girbons/comics-downloader/pkg/config" + "github.com/Girbons/comics-downloader/pkg/util" + "github.com/beevik/etree" +) type ComicOutputFormat string @@ -32,3 +45,166 @@ func ToComicOutputFormat(format string) (ComicOutputFormat, error) { return "", InvalidOutputFormatError } } + +// makeComicInfoXML generates a ComicInfo.xml file for the given comic issue and saves it to the output directory. It returns the path to the generated ComicInfo.xml file. +// Based on the ComicInfo.xml https://anansi-project.github.io/docs/comicinfo/schemas/v2.1 +func (comic *ComicIssue) makeComicInfoXML(options *config.Options, images *DownloadResult) (string, error) { + outputDir, err := util.ImagesPathSetup(options.CreateDefaultPath, options.OutputFolder, comic.Source.Name, comic.Name, options.IssueFolderName, comic.IssueNumber) + if err != nil { + return "", err + } + + comicInfoPath := filepath.Join(outputDir, "ComicInfo.xml") + options.Logger.Infof("ComicInfo.xml path: %s", comicInfoPath) + + fo, err := os.Create(comicInfoPath) + if err != nil { + return "", err + } + defer func() { + if err := fo.Close(); err != nil { + panic(err) + } + }() + + doc := etree.NewDocument() + doc.CreateProcInst("xml", `version="1.0" encoding="UTF-8"`) + + comicInfo := doc.CreateElement("ComicInfo") + + comicInfo.CreateAttr("xmlns:xsi", "http://www.w3.org/2001/XMLSchema-instance") + comicInfo.CreateAttr("xsi:noNamespaceSchemaLocation", "https://github.com/anansi-project/comicinfo/raw/db8e1d84132f97403b226f2e12aaec1342c2a223/drafts/v2.1/ComicInfo.xsd") + comicInfo.CreateElement("Notes").SetText(fmt.Sprintf("Tagged by comics-downloader version %s using info from %s at %s", version.Tag, comic.Source.Name, time.Now().Format(time.RFC3339))) + + comicInfo.CreateElement("Series").SetText(comic.SeriesMetadata.Title) + comicInfo.CreateElement("Title").SetText(comic.Name) + for lang, localizedTitle := range comic.SeriesMetadata.LocalizedTitle { + if lang == "en" { + // non-standard field + comicInfo.CreateElement("LocalizedSeries").SetText(localizedTitle) + break + } + } + for lang, localizedDesc := range comic.SeriesMetadata.Description { + if lang == "en" { + comicInfo.CreateElement("Summary").SetText(localizedDesc) + break + } + } + + comicInfo.CreateElement("Number").SetText(comic.IssueNumber) + if comic.Volume != nil { + comicInfo.CreateElement("Volume").SetText(*comic.Volume) + } + + if comic.ReleaseDate != nil { + comicInfo.CreateElement("Year").SetText(fmt.Sprintf("%d", comic.ReleaseDate.Year())) + comicInfo.CreateElement("Month").SetText(fmt.Sprintf("%02d", comic.ReleaseDate.Month())) + comicInfo.CreateElement("Day").SetText(fmt.Sprintf("%02d", comic.ReleaseDate.Day())) + } + if comic.LanguageISO != nil { + comicInfo.CreateElement("LanguageISO").SetText(*comic.LanguageISO) + } + + if comic.SeriesMetadata.IsManga != nil { + // TODO: consider adding unknown value for manga field if IsManga is nil? + mangaTag := comicInfo.CreateElement("Manga") + if *comic.SeriesMetadata.IsManga { + if comic.SeriesMetadata.IsRTL != nil && *comic.SeriesMetadata.IsRTL { + mangaTag.SetText("YesAndRightToLeft") + } else { + mangaTag.SetText("Yes") + } + } else { + mangaTag.SetText("No") + } + } + if comic.SeriesMetadata.AgeRating != nil { + comicInfo.CreateElement("AgeRating").SetText(string(*comic.SeriesMetadata.AgeRating)) + } + if comic.SeriesMetadata.CommunityRating != nil { + comicInfo.CreateElement("CommunityRating").SetText(fmt.Sprintf("%.2f", *comic.SeriesMetadata.CommunityRating)) + } + if len(comic.SeriesMetadata.Tags) > 0 { + comicInfo.CreateElement("Tags").SetText(strings.Join(comic.SeriesMetadata.Tags, ",")) + } + if len(comic.SeriesMetadata.Genres) > 0 { + comicInfo.CreateElement("Genres").SetText(strings.Join(comic.SeriesMetadata.Genres, ",")) + } + if len(comic.SeriesMetadata.WebLinks) > 0 { + var cleanedWebLinks []string + for _, link := range comic.SeriesMetadata.WebLinks { + // the links must be URL-encoded as spaces are the separator + cleanedWebLinks = append(cleanedWebLinks, url.QueryEscape(link)) + } + comicInfo.CreateElement("WebLinks").SetText(strings.Join(cleanedWebLinks, " ")) + } + if comic.SeriesMetadata.AgeRating != nil { + comicInfo.CreateElement("AgeRating").SetText(string(*comic.SeriesMetadata.AgeRating)) + } + if len(comic.SeriesMetadata.Creators) > 0 { + var writers []string + var pencillers []string + var inkers []string + var colorists []string + var letterers []string + var coverArtists []string + var editors []string + var translators []string + + for _, creator := range comic.SeriesMetadata.Creators { + switch creator.Role { + case CreatorRoleWriter: + writers = append(writers, creator.Name) + case CreatorRolePenciller: + pencillers = append(pencillers, creator.Name) + case CreatorRoleInker: + inkers = append(inkers, creator.Name) + case CreatorRoleColorist: + colorists = append(colorists, creator.Name) + case CreatorRoleLetterer: + letterers = append(letterers, creator.Name) + case CreatorRoleCoverArtist: + coverArtists = append(coverArtists, creator.Name) + case CreatorRoleEditor: + editors = append(editors, creator.Name) + case CreatorRoleTranslator: + translators = append(translators, creator.Name) + } + // TODO: handle unknown roles + } + if len(writers) > 0 { + comicInfo.CreateElement("Writer").SetText(strings.Join(writers, ",")) + } + if len(pencillers) > 0 { + comicInfo.CreateElement("Penciller").SetText(strings.Join(pencillers, ",")) + } + if len(inkers) > 0 { + comicInfo.CreateElement("Inker").SetText(strings.Join(inkers, ",")) + } + if len(colorists) > 0 { + comicInfo.CreateElement("Colorist").SetText(strings.Join(colorists, ",")) + } + if len(letterers) > 0 { + comicInfo.CreateElement("Letterer").SetText(strings.Join(letterers, ",")) + } + if len(coverArtists) > 0 { + comicInfo.CreateElement("CoverArtist").SetText(strings.Join(coverArtists, ",")) + } + if len(editors) > 0 { + comicInfo.CreateElement("Editor").SetText(strings.Join(editors, ",")) + } + if len(translators) > 0 { + comicInfo.CreateElement("Translator").SetText(strings.Join(translators, ",")) + } + } + comicInfo.CreateElement("PageCount").SetText(fmt.Sprintf("%d", len(images.FilePaths))) + + doc.Indent(2) + _, err = doc.WriteTo(fo) + if err != nil { + return "", err + } + + return comicInfoPath, nil +} diff --git a/pkg/sites/loader.go b/pkg/sites/loader.go index 394956ab..fc8f617f 100644 --- a/pkg/sites/loader.go +++ b/pkg/sites/loader.go @@ -58,12 +58,17 @@ func initializeCollection(issues []string, options *config.Options, base BaseSit comic := &core.ComicIssue{ Name: name, IssueNumber: issueNumber, + + OutputFormat: outputFormat, + ImagesFormat: options.ImagesFormat, + Source: &core.ComicSource{ Name: options.SourceName, URL: url, }, - OutputFormat: outputFormat, - ImagesFormat: options.ImagesFormat, + SeriesMetadata: &core.SeriesMetadata{ + Title: name, + }, } if err = base.Initialize(comic); err != nil { return collection, err From 35102aa87e491e25594c0a6c15837159c72516e1 Mon Sep 17 00:00:00 2001 From: ProjectDislocate <173194962+ProjectDislocate@users.noreply.github.com> Date: Fri, 6 Mar 2026 14:07:34 -0500 Subject: [PATCH 21/79] feat: improve mangadex metadata quality --- pkg/sites/mangadex.go | 103 ++++++++++++++++++++++++++++++------- pkg/sites/mangadex_test.go | 2 +- 2 files changed, 85 insertions(+), 20 deletions(-) diff --git a/pkg/sites/mangadex.go b/pkg/sites/mangadex.go index f099e13b..fc89aa44 100644 --- a/pkg/sites/mangadex.go +++ b/pkg/sites/mangadex.go @@ -139,8 +139,22 @@ func (m *Mangadex) getChapters(mangaID string) ([]string, error) { return ids, nil } +type mangadexChapter struct { + ChapterID string + ChapterNumber string + ChapterTitle string + Volume *string // can be null if not available + + TranslatedLanguage string // Not sure if this always exists or not e.g. "en", "jp" + PublishAt time.Time + + MangaID string + + ImageLinks []string +} + // getChapter retrieves metadata and image links for a single chapter. -func (m *Mangadex) getChapter(chapterID string) (mangaID, volume, chapter, title string, images []string, err error) { +func (m *Mangadex) getChapter(chapterID string) (chapterInfo mangadexChapter, err error) { ctx, cancel := m.requestContext() defer cancel() @@ -148,10 +162,14 @@ func (m *Mangadex) getChapter(chapterID string) (mangaID, volume, chapter, title var chapterRes struct { Result string `json:"result"` Data struct { + ID string `json:"id"` // chapter ID + Type string `json:"type"` // should be "chapter" Attributes struct { - Volume string `json:"volume"` - Chapter string `json:"chapter"` - Title string `json:"title"` + Volume *string `json:"volume"` // is null if not available + Chapter *string `json:"chapter"` + Title *string `json:"title"` + TranslatedLanguage string `json:"translatedLanguage"` + PublishAt string `json:"publishAt"` } `json:"attributes"` Relationships []struct { ID string `json:"id"` @@ -161,10 +179,15 @@ func (m *Mangadex) getChapter(chapterID string) (mangaID, volume, chapter, title } if err := fetchJSON(ctx, m.client, endpoint, &chapterRes); err != nil { - return "", "", "", "", nil, err + return mangadexChapter{}, err } if strings.ToLower(chapterRes.Result) != "ok" { - return "", "", "", "", nil, fmt.Errorf("unexpected response") + return mangadexChapter{}, fmt.Errorf("unexpected response") + } + + publishedAt, err := time.Parse(time.RFC3339, chapterRes.Data.Attributes.PublishAt) + if err != nil { + return mangadexChapter{}, err } imagesEndpoint := joinURL(m.apiBase, fmt.Sprintf("/at-home/server/%s", chapterID)) @@ -177,21 +200,23 @@ func (m *Mangadex) getChapter(chapterID string) (mangaID, volume, chapter, title } if err := fetchJSON(ctx, m.client, imagesEndpoint, &imagesRes); err != nil { - return "", "", "", "", nil, err + return mangadexChapter{}, err } if strings.ToLower(imagesRes.Result) != "ok" { - return "", "", "", "", nil, fmt.Errorf("unexpected response") + return mangadexChapter{}, fmt.Errorf("unexpected response") } + var imageLinks []string for _, file := range imagesRes.Chapter.Data { imageURL := joinURL(m.uploadsBase, fmt.Sprintf("%s/%s", imagesRes.Chapter.Hash, file)) - images = append(images, imageURL) + imageLinks = append(imageLinks, imageURL) } - if m.options.Debug && len(images) > 0 && m.options.Logger != nil { - m.options.Logger.Debug(fmt.Sprintf("Image Links found: %s", strings.Join(images, " "))) + if m.options.Debug && len(imageLinks) > 0 && m.options.Logger != nil { + m.options.Logger.Debug(fmt.Sprintf("Image Links found: %s", strings.Join(imageLinks, " "))) } + var mangaID string for _, rel := range chapterRes.Data.Relationships { if rel.Type == "manga" { mangaID = rel.ID @@ -199,7 +224,30 @@ func (m *Mangadex) getChapter(chapterID string) (mangaID, volume, chapter, title } } - return mangaID, chapterRes.Data.Attributes.Volume, chapterRes.Data.Attributes.Chapter, chapterRes.Data.Attributes.Title, images, nil + // just default them to empty string if not found + var chapterNumber string + var chapterTitle string + if chapterRes.Data.Attributes.Chapter != nil { + // TODO: consider defaulting to "oneshot"? + chapterNumber = *chapterRes.Data.Attributes.Chapter + } + if chapterRes.Data.Attributes.Title != nil { + chapterTitle = *chapterRes.Data.Attributes.Title + } + + return mangadexChapter{ + ChapterID: chapterRes.Data.ID, + ChapterNumber: chapterNumber, + ChapterTitle: chapterTitle, + Volume: chapterRes.Data.Attributes.Volume, + + TranslatedLanguage: chapterRes.Data.Attributes.TranslatedLanguage, + PublishAt: publishedAt, + + MangaID: mangaID, + + ImageLinks: imageLinks, + }, nil } // RetrieveIssueLinks retrieve the issue links for the given comic. @@ -226,19 +274,28 @@ func (m *Mangadex) GetInfo(urlValue string) (string, string) { } switch parts[3] { case "chapter": - mangaID, volume, chapter, title, _, err := m.getChapter(parts[4]) + chapter, err := m.getChapter(parts[4]) if err != nil { return "", "" } - chapterTitle := fmt.Sprintf("Vol %s Chapter %s", volume, chapter) - if title != "" { - chapterTitle += fmt.Sprintf(", %s", title) + + var chapterTitle string + if chapter.Volume != nil { + volume := *chapter.Volume + chapterTitle = fmt.Sprintf("Vol %s Chapter %s", volume, chapter.ChapterNumber) + } else { + chapterTitle = fmt.Sprintf("Chapter %s", chapter.ChapterNumber) + } + + if chapter.ChapterTitle != "" { + chapterTitle += fmt.Sprintf(", %s", chapter.ChapterTitle) } - mangaTitle, err := m.getManga(mangaID) + mangaTitle, err := m.getManga(chapter.MangaID) if err != nil { return "", chapterTitle } return mangaTitle, chapterTitle + case "title": mangaTitle, err := m.getManga(parts[4]) if err != nil { @@ -256,10 +313,18 @@ func (m *Mangadex) Initialize(comic *core.ComicIssue) error { if len(parts) < 5 { return fmt.Errorf("URL not supported") } - _, _, _, _, images, err := m.getChapter(parts[4]) + chapter, err := m.getChapter(parts[4]) if err != nil { return err } - comic.ImageLinks = images + + // comic.Name = chapter.ChapterTitle // changing the title seems to break path resolving for some reason, probably because the folder has already been created by the time we get to this point, so we just keep the name as is until the metadata system is reworked + comic.IssueNumber = chapter.ChapterNumber + comic.Volume = chapter.Volume + comic.LanguageISO = &chapter.TranslatedLanguage + comic.ReleaseDate = &chapter.PublishAt + + comic.ImageLinks = chapter.ImageLinks + return nil } diff --git a/pkg/sites/mangadex_test.go b/pkg/sites/mangadex_test.go index 2060ed3b..09fda8cb 100644 --- a/pkg/sites/mangadex_test.go +++ b/pkg/sites/mangadex_test.go @@ -44,7 +44,7 @@ func setupMangadexServer() *httptest.Server { fmt.Fprint(w, `{ "result":"ok", "data":{ - "attributes":{"volume":"1","chapter":"1","title":"Start"}, + "attributes":{"volume":"1","chapter":"1","title":"Start","publishAt":"2026-03-06T14:03:52.000Z","translatedLanguage":"en"}, "relationships":[{"id":"series-1","type":"manga"}] } }`) From 3fd623eaca87b69e4ee12638dec03c50dd5a2f5a Mon Sep 17 00:00:00 2001 From: ProjectDislocate <173194962+ProjectDislocate@users.noreply.github.com> Date: Fri, 6 Mar 2026 14:19:22 -0500 Subject: [PATCH 22/79] fix: missing assert import --- pkg/sites/loader_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/sites/loader_test.go b/pkg/sites/loader_test.go index 643d4364..dd2c1407 100644 --- a/pkg/sites/loader_test.go +++ b/pkg/sites/loader_test.go @@ -6,6 +6,7 @@ import ( "github.com/Girbons/comics-downloader/pkg/config" "github.com/Girbons/comics-downloader/pkg/core" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) From 8808fa668fcc151482cdd196e2e584ecf44dbd1a Mon Sep 17 00:00:00 2001 From: ProjectDislocate <173194962+ProjectDislocate@users.noreply.github.com> Date: Fri, 6 Mar 2026 16:58:36 -0500 Subject: [PATCH 23/79] feat: customizable request timeout for all providers --- cmd/downloader/main.go | 7 ++- go.mod | 2 +- go.sum | 4 +- pkg/config/options.go | 15 ++++-- pkg/core/output.go | 2 + pkg/sites/comicextra.go | 27 +++++++--- pkg/sites/comicextra_test.go | 19 ++++--- pkg/sites/common.go | 51 +++++++++++++------ ...eobfuscate_test.go => deobfuscate_test.go} | 0 pkg/sites/http_helpers.go | 8 +++ pkg/sites/loader.go | 6 +++ pkg/sites/mangadex.go | 27 ++++------ pkg/sites/mangadex_test.go | 11 ++-- pkg/sites/mangakakalot.go | 12 +++-- pkg/sites/mangakakalot_test.go | 7 +-- pkg/sites/manganato.go | 12 +++-- pkg/sites/manganato_test.go | 7 +-- pkg/sites/mangareader.go | 23 ++++++--- pkg/sites/mangareader_test.go | 5 +- pkg/sites/mangatown.go | 28 +++++++--- pkg/sites/mangatown_test.go | 5 +- pkg/sites/readallcomics.go | 20 ++++++-- pkg/sites/readallcomics_test.go | 5 +- pkg/sites/readcomiconline.go | 23 ++++++--- pkg/sites/readcomiconline_test.go | 5 +- 25 files changed, 225 insertions(+), 106 deletions(-) rename pkg/sites/{comicextra_deobfuscate_test.go => deobfuscate_test.go} (100%) diff --git a/cmd/downloader/main.go b/cmd/downloader/main.go index 55e7fdad..a4928cea 100644 --- a/cmd/downloader/main.go +++ b/cmd/downloader/main.go @@ -45,8 +45,9 @@ var ( // string to be used for each issue/chapter folder issueFolderName string // request customization - userAgentsCSV string - sessionCookie string + userAgentsCSV string + sessionCookie string + requestTimeout time.Duration // throttling requestDelay time.Duration requestDelayJitter time.Duration @@ -72,6 +73,7 @@ func init() { flag.StringVar(&issueFolderName, "issue-folder-name", "issue-", "Folder name where each issue/chapter will be saved, default 'issue-#'") flag.StringVar(&userAgentsCSV, "user-agents", "", "Comma-separated list of alternative User-Agent values to rotate per request") flag.StringVar(&sessionCookie, "session-cookie", "", "Custom Cookie header value (e.g., cf_clearance=...; other=...) for protected sources") + flag.DurationVar(&requestTimeout, "request-timeout", config.DefaulltRequestTimeout, "Timeout for HTTP requests (e.g., 8s)") flag.DurationVar(&requestDelay, "request-delay", config.DefaultRequestDelay, "Base delay inserted before downloading each image (e.g. 500ms)") flag.DurationVar(&requestDelayJitter, "request-delay-jitter", config.DefaultRequestDelayJitter, "Maximum additional random delay added to the base request delay (e.g. 250ms)") @@ -99,6 +101,7 @@ func buildOptions() config.Options { IssueFolderName: issueFolderName, UserAgents: splitAndTrim(userAgentsCSV), SessionCookie: strings.TrimSpace(sessionCookie), + RequestTimeout: requestTimeout, RequestDelay: requestDelay, RequestDelayJitter: requestDelayJitter, } diff --git a/go.mod b/go.mod index 7f3c6ec9..77eeb95a 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/mholt/archives v0.1.2 github.com/schollz/progressbar/v2 v2.15.0 github.com/sirupsen/logrus v1.9.3 - github.com/stretchr/testify v1.9.0 + github.com/stretchr/testify v1.11.1 golang.org/x/image v0.18.0 golang.org/x/mod v0.17.0 golang.org/x/sync v0.12.0 diff --git a/go.sum b/go.sum index 3738e72d..c96321e6 100644 --- a/go.sum +++ b/go.sum @@ -175,8 +175,8 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= -github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/therootcompany/xz v1.0.1 h1:CmOtsn1CbtmyYiusbfmhmkpAAETj0wBIH6kCYaX+xzw= github.com/therootcompany/xz v1.0.1/go.mod h1:3K3UH1yCKgBneZYhuQUvJ9HPD19UEXEI0BWbMn8qNMY= github.com/ulikunitz/xz v0.5.8/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= diff --git a/pkg/config/options.go b/pkg/config/options.go index a46846fd..52ecbf0d 100644 --- a/pkg/config/options.go +++ b/pkg/config/options.go @@ -12,6 +12,8 @@ const ( DefaultRequestDelay = 500 * time.Millisecond // DefaultRequestDelayJitter adds up to this much random extra delay to avoid fixed patterns. DefaultRequestDelayJitter = 250 * time.Millisecond + // DefaultRequestTimeout is the default timeout for HTTP requests. + DefaulltRequestTimeout = 30 * time.Second ) // Options represents the comics downloader options. @@ -34,11 +36,16 @@ type Options struct { SourceName string IssuesRange string IssueFolderName string - UserAgents []string - SessionCookie string - RequestDelay time.Duration - RequestDelayJitter time.Duration + + UserAgents []string + SessionCookie string + RequestDelay time.Duration + RequestDelayJitter time.Duration + RequestTimeout time.Duration Client *http.ComicClient Logger *logger.Logger } + +// TODO: create function to handle creating options with default values and factories for client and logger +// want to avoid having to avoid malformed options diff --git a/pkg/core/output.go b/pkg/core/output.go index 72bd0a20..2997eeab 100644 --- a/pkg/core/output.go +++ b/pkg/core/output.go @@ -79,6 +79,7 @@ func (comic *ComicIssue) makeComicInfoXML(options *config.Options, images *Downl comicInfo.CreateElement("Series").SetText(comic.SeriesMetadata.Title) comicInfo.CreateElement("Title").SetText(comic.Name) for lang, localizedTitle := range comic.SeriesMetadata.LocalizedTitle { + // TODO: use the country option to select the localized title instead of defaulting to English, or add a separate option for the localized title language if lang == "en" { // non-standard field comicInfo.CreateElement("LocalizedSeries").SetText(localizedTitle) @@ -86,6 +87,7 @@ func (comic *ComicIssue) makeComicInfoXML(options *config.Options, images *Downl } } for lang, localizedDesc := range comic.SeriesMetadata.Description { + // TODO: use the country option to select the localized description instead of defaulting to English, or add a separate option for the localized description language if lang == "en" { comicInfo.CreateElement("Summary").SetText(localizedDesc) break diff --git a/pkg/sites/comicextra.go b/pkg/sites/comicextra.go index 7f80cb75..58d0636a 100644 --- a/pkg/sites/comicextra.go +++ b/pkg/sites/comicextra.go @@ -1,6 +1,7 @@ package sites import ( + "context" "fmt" "regexp" "sort" @@ -8,6 +9,7 @@ import ( "github.com/Girbons/comics-downloader/pkg/config" "github.com/Girbons/comics-downloader/pkg/core" + httpclient "github.com/Girbons/comics-downloader/pkg/http" "github.com/Girbons/comics-downloader/pkg/util" "github.com/anaskhan96/soup" ) @@ -15,19 +17,26 @@ import ( // Comicextra represents comicextra instance. type Comicextra struct { options *config.Options + client *httpclient.ComicClient } // NewComicextra returs a comicextra instance. func NewComicextra(options *config.Options) *Comicextra { return &Comicextra{ options: options, + client: options.Client, } } +func (c *Comicextra) requestContext() (context.Context, context.CancelFunc) { + return context.WithTimeout(context.Background(), c.options.RequestTimeout) +} + func (c *Comicextra) retrieveImageLinks(comic *core.ComicIssue) ([]string, error) { - var links []string + ctx, cancel := c.requestContext() + defer cancel() - response, err := soup.Get(comic.Source.URL) + response, err := fetchHTML(ctx, c.client, comic.Source.URL) if err != nil { return nil, err } @@ -35,6 +44,7 @@ func (c *Comicextra) retrieveImageLinks(comic *core.ComicIssue) ([]string, error re := regexp.MustCompile(util.IMAGEREGEX) match := re.FindAllStringSubmatch(response, -1) + var links []string for i := range match { link := deobfuscateURL(match[i][1]) if util.IsURLValid(link) { @@ -54,10 +64,10 @@ func (c *Comicextra) isSingleIssue(url string) bool { } func (c *Comicextra) retrieveLastIssue(url string) (string, error) { - var lastIssue string - - response, err := soup.Get(url) + ctx, cancel := c.requestContext() + defer cancel() + response, err := fetchHTML(ctx, c.client, url) if err != nil { return "", err } @@ -77,7 +87,7 @@ func (c *Comicextra) retrieveLastIssue(url string) (string, error) { sort.Strings(validLinks) - lastIssue = validLinks[len(validLinks)-1] + lastIssue := validLinks[len(validLinks)-1] return lastIssue, nil } @@ -118,7 +128,10 @@ func (c *Comicextra) RetrieveIssueLinks() ([]string, error) { elements []soup.Root ) - response, err := soup.Get(url) + ctx, cancel := c.requestContext() + defer cancel() + + response, err := fetchHTML(ctx, c.client, url) if err != nil { return nil, err } diff --git a/pkg/sites/comicextra_test.go b/pkg/sites/comicextra_test.go index 830e7d5e..b058e134 100644 --- a/pkg/sites/comicextra_test.go +++ b/pkg/sites/comicextra_test.go @@ -63,8 +63,9 @@ func TestComicExtraScraper(t *testing.T) { defer server.Close() opts := &config.Options{ - URL: server.URL + comicExtraIssueFullPath, - Logger: logger.NewLogger(false, nil), + URL: server.URL + comicExtraIssueFullPath, + Logger: logger.NewLogger(false, nil), + RequestTimeout: config.DefaulltRequestTimeout, } comicextra := NewComicextra(opts) @@ -87,9 +88,10 @@ func TestComicExtraRetrieveIssueLinksAll(t *testing.T) { defer server.Close() opts := &config.Options{ - URL: server.URL + comicExtraListPath, - All: true, - Logger: logger.NewLogger(false, nil), + URL: server.URL + comicExtraListPath, + All: true, + Logger: logger.NewLogger(false, nil), + RequestTimeout: config.DefaulltRequestTimeout, } comicextra := NewComicextra(opts) @@ -106,9 +108,10 @@ func TestComicExtraRetrieveLastIssue(t *testing.T) { defer server.Close() opts := &config.Options{ - URL: server.URL + comicExtraLastIssuePath, - Last: true, - Logger: logger.NewLogger(false, nil), + URL: server.URL + comicExtraLastIssuePath, + Last: true, + Logger: logger.NewLogger(false, nil), + RequestTimeout: config.DefaulltRequestTimeout, } comicextra := NewComicextra(opts) diff --git a/pkg/sites/common.go b/pkg/sites/common.go index 945270bf..39711260 100644 --- a/pkg/sites/common.go +++ b/pkg/sites/common.go @@ -1,8 +1,10 @@ package sites import ( + "context" "strings" + "github.com/Girbons/comics-downloader/pkg/config" "github.com/Girbons/comics-downloader/pkg/core" "github.com/Girbons/comics-downloader/pkg/util" "github.com/anaskhan96/soup" @@ -11,12 +13,20 @@ import ( // mangakakalot.com and manganato.com functions -func MangaKakalotGetInfo(domain string, url string) (string, string) { - // get chapter name - res, err := soup.Get(url) +func mangaKakalotRequestContext(options *config.Options) (context.Context, context.CancelFunc) { + return context.WithTimeout(context.Background(), options.RequestTimeout) +} + +func MangaKakalotGetInfo(options *config.Options, domain string, url string) (name, issueNumber string, err error) { + ctx, cancel := mangaKakalotRequestContext(options) + defer cancel() + + res, err := fetchHTML(ctx, options.Client, url) if err != nil { - return "", "" + return "", "", err } + + // get chapter name doc := soup.HTMLParse(res) f := doc.Find("div", "class", breadcrumbClassName(domain)) switch { @@ -29,22 +39,26 @@ func MangaKakalotGetInfo(domain string, url string) (string, string) { items := f.FindAll("a", "class", "a-h") f = items[len(items)-1] } - name := f.Text() + name = f.Text() name, err = regexp2.MustCompile("(Vol\\.[0-9]{1,3} )?(Chapter [0-9]{1,3}(\\.[0-9])?) ?: ", 0).Replace(name, "", 0, 1) if err != nil { - return "", "" + return "", "", err } // parse number from url parts := util.TrimAndSplitURL(url) - issueNumber := strings.Split(parts[len(parts)-1], "-")[1] - return name, issueNumber + issueNumber = strings.Split(parts[len(parts)-1], "-")[1] + return name, issueNumber, nil } -func MangaKakalotInitialize(comic *core.ComicIssue) error { - res, err := soup.Get(comic.Source.URL) +func MangaKakalotInitialize(options *config.Options, comic *core.ComicIssue) error { + ctx, cancel := mangaKakalotRequestContext(options) + defer cancel() + + res, err := fetchHTML(ctx, options.Client, comic.Source.URL) if err != nil { return err } + doc := soup.HTMLParse(res) f := doc.Find("div", "class", "container-chapter-reader") var links []string @@ -55,15 +69,20 @@ func MangaKakalotInitialize(comic *core.ComicIssue) error { return nil } -func MangaKakalotRetrieveIssueLinks(domain string, url string) ([]string, error) { - res, err := soup.Get(url) - if err != nil { - panic(err) - } - // chapter page link +func MangaKakalotRetrieveIssueLinks(options *config.Options, domain string, url string) ([]string, error) { + // if chapter page, skip fetching and parsing the list page if strings.Contains(url, "/chapter") { return []string{url}, nil } + + ctx, cancel := mangaKakalotRequestContext(options) + defer cancel() + + res, err := fetchHTML(ctx, options.Client, url) + if err != nil { + return nil, err + } + // manga page link doc := soup.HTMLParse(res) f := doc.Find("div", "class", chapterListClassName(domain)) diff --git a/pkg/sites/comicextra_deobfuscate_test.go b/pkg/sites/deobfuscate_test.go similarity index 100% rename from pkg/sites/comicextra_deobfuscate_test.go rename to pkg/sites/deobfuscate_test.go diff --git a/pkg/sites/http_helpers.go b/pkg/sites/http_helpers.go index 0b4d202b..e9e29d8a 100644 --- a/pkg/sites/http_helpers.go +++ b/pkg/sites/http_helpers.go @@ -38,6 +38,14 @@ func buildRequest(ctx context.Context, client *httpclient.ComicClient, link stri return req, nil } +func fetchHTML(ctx context.Context, client *httpclient.ComicClient, link string) (string, error) { + response, err := fetchBytes(ctx, client, link) + if err != nil { + return "", err + } + return string(response), nil +} + func fetchJSON(ctx context.Context, client *httpclient.ComicClient, link string, target interface{}) error { if target == nil { return fmt.Errorf("target cannot be nil") diff --git a/pkg/sites/loader.go b/pkg/sites/loader.go index fe7f472e..950d95d3 100644 --- a/pkg/sites/loader.go +++ b/pkg/sites/loader.go @@ -9,6 +9,7 @@ import ( "github.com/Girbons/comics-downloader/internal/flag/parser" "github.com/Girbons/comics-downloader/pkg/config" "github.com/Girbons/comics-downloader/pkg/core" + httpclient "github.com/Girbons/comics-downloader/pkg/http" "github.com/Girbons/comics-downloader/pkg/util" ) @@ -135,6 +136,11 @@ func LoadComicFromSource(options *config.Options) ([]*core.ComicIssue, error) { err error ) + // ensure the client is actually set + if options.Client == nil { + options.Client = httpclient.NewComicClient() + } + switch { case strings.Contains(options.SourceName, "readcomiconline"): base = NewReadComiconline(options) diff --git a/pkg/sites/mangadex.go b/pkg/sites/mangadex.go index fc89aa44..1e0e6bee 100644 --- a/pkg/sites/mangadex.go +++ b/pkg/sites/mangadex.go @@ -16,10 +16,9 @@ import ( ) const ( - mangadexAPIBase = "https://api.mangadex.org" - mangadexChapterBase = "https://mangadex.org/chapter" - mangadexUploadsBase = "https://uploads.mangadex.org/data" - mangadexRequestTimeout = 8 * time.Second + mangadexAPIBase = "https://api.mangadex.org" + mangadexChapterBase = "https://mangadex.org/chapter" + mangadexUploadsBase = "https://uploads.mangadex.org/data" ) // Mangadex represents a mangadex instance. @@ -34,16 +33,10 @@ type Mangadex struct { // NewMangadex returns a Mangadex instance. func NewMangadex(options *config.Options) *Mangadex { - client := options.Client - if client == nil { - client = httpclient.NewComicClient() - options.Client = client - } - return &Mangadex{ country: strings.ToLower(options.Country), options: options, - client: client, + client: options.Client, apiBase: mangadexAPIBase, chapterBase: mangadexChapterBase, uploadsBase: mangadexUploadsBase, @@ -51,7 +44,7 @@ func NewMangadex(options *config.Options) *Mangadex { } func (m *Mangadex) requestContext() (context.Context, context.CancelFunc) { - return context.WithTimeout(context.Background(), mangadexRequestTimeout) + return context.WithTimeout(context.Background(), m.options.RequestTimeout) } func joinURL(base, suffix string) string { @@ -79,15 +72,17 @@ func (m *Mangadex) getManga(mangaID string) (string, error) { return "", fmt.Errorf("unexpected response") } - for lang, t := range mangaRes.Data.Attributes.Titles { + // TODO: set localized title based on country option instead of setting main title to that language + // then need to update how paths are generated to use the localized title instead of the main title if the country option is set + for lang, title := range mangaRes.Data.Attributes.Titles { if m.country == "" || m.country == strings.ToLower(lang) { - return t, nil + return title, nil } } // Fallback to any available title. - for _, t := range mangaRes.Data.Attributes.Titles { - return t, nil + for _, title := range mangaRes.Data.Attributes.Titles { + return title, nil } return "", fmt.Errorf("no title found for manga %s", mangaID) diff --git a/pkg/sites/mangadex_test.go b/pkg/sites/mangadex_test.go index 09fda8cb..6f2207ec 100644 --- a/pkg/sites/mangadex_test.go +++ b/pkg/sites/mangadex_test.go @@ -71,11 +71,12 @@ func newTestMangadex(t *testing.T) (*Mangadex, func()) { ) opts := &config.Options{ - URL: server.URL + "/title/series-1/naruto", - Country: "en", - SourceName: "mangadex.org", - Logger: logger.NewLogger(false, nil), - Client: client, + URL: server.URL + "/title/series-1/naruto", + Country: "en", + SourceName: "mangadex.org", + Logger: logger.NewLogger(false, nil), + Client: client, + RequestTimeout: config.DefaulltRequestTimeout, } md := NewMangadex(opts) diff --git a/pkg/sites/mangakakalot.go b/pkg/sites/mangakakalot.go index 616e5c0d..890b48fb 100644 --- a/pkg/sites/mangakakalot.go +++ b/pkg/sites/mangakakalot.go @@ -18,15 +18,21 @@ func NewMangaKakalot(options *config.Options) *MangaKakalot { // GetInfo extracts the basic info from the given url. func (m *MangaKakalot) GetInfo(url string) (string, string) { - return MangaKakalotGetInfo("mangakakalot.com", url) + name, issueNumber, err := MangaKakalotGetInfo(m.options, "mangakakalot.com", url) + if err != nil { + m.options.Logger.Errorf("error getting info for url %q: %v", url, err) + return "", "" + } + + return name, issueNumber } // Initialize loads links and metadata from mangakakalot func (m *MangaKakalot) Initialize(comic *core.ComicIssue) error { - return MangaKakalotInitialize(comic) + return MangaKakalotInitialize(m.options, comic) } // RetrieveIssueLinks retrieve the issue links for the given comic. func (m *MangaKakalot) RetrieveIssueLinks() ([]string, error) { - return MangaKakalotRetrieveIssueLinks("mangakakalot.com", m.options.URL) + return MangaKakalotRetrieveIssueLinks(m.options, "mangakakalot.com", m.options.URL) } diff --git a/pkg/sites/mangakakalot_test.go b/pkg/sites/mangakakalot_test.go index ac8eabeb..be5b4e74 100644 --- a/pkg/sites/mangakakalot_test.go +++ b/pkg/sites/mangakakalot_test.go @@ -69,9 +69,10 @@ func TestMangaKakalotScraper(t *testing.T) { defer server.Close() opts := &config.Options{ - URL: server.URL + mangaKakalotListPath, - SourceName: "mangakakalot.com", - Logger: logger.NewLogger(false, nil), + URL: server.URL + mangaKakalotListPath, + SourceName: "mangakakalot.com", + Logger: logger.NewLogger(false, nil), + RequestTimeout: config.DefaulltRequestTimeout, } scraper := NewMangaKakalot(opts) diff --git a/pkg/sites/manganato.go b/pkg/sites/manganato.go index 08cb1233..7c53b9a9 100644 --- a/pkg/sites/manganato.go +++ b/pkg/sites/manganato.go @@ -18,15 +18,21 @@ func NewManganato(options *config.Options) *Manganato { // GetInfo extracts the basic info from the given url. func (m *Manganato) GetInfo(url string) (string, string) { - return MangaKakalotGetInfo("manganato.com", url) + name, issueNumber, err := MangaKakalotGetInfo(m.options, "manganato.com", url) + if err != nil { + m.options.Logger.Errorf("error getting info for url %q: %v", url, err) + return "", "" + } + + return name, issueNumber } // Initialize loads links and metadata from manganato func (m *Manganato) Initialize(comic *core.ComicIssue) error { - return MangaKakalotInitialize(comic) + return MangaKakalotInitialize(m.options, comic) } // RetrieveIssueLinks retrieve the issue links for the given comic. func (m *Manganato) RetrieveIssueLinks() ([]string, error) { - return MangaKakalotRetrieveIssueLinks("manganato.com", m.options.URL) + return MangaKakalotRetrieveIssueLinks(m.options, "manganato.com", m.options.URL) } diff --git a/pkg/sites/manganato_test.go b/pkg/sites/manganato_test.go index 3719efb0..0e7f9ca9 100644 --- a/pkg/sites/manganato_test.go +++ b/pkg/sites/manganato_test.go @@ -66,9 +66,10 @@ func TestManganatoScraper(t *testing.T) { defer server.Close() opts := &config.Options{ - URL: server.URL + manganatoListPath, - SourceName: "manganato.com", - Logger: logger.NewLogger(false, nil), + URL: server.URL + manganatoListPath, + SourceName: "manganato.com", + Logger: logger.NewLogger(false, nil), + RequestTimeout: config.DefaulltRequestTimeout, } scraper := NewManganato(opts) diff --git a/pkg/sites/mangareader.go b/pkg/sites/mangareader.go index 32769498..4e3767f9 100644 --- a/pkg/sites/mangareader.go +++ b/pkg/sites/mangareader.go @@ -1,6 +1,7 @@ package sites import ( + "context" "fmt" "strings" @@ -22,15 +23,20 @@ func NewMangareader(options *config.Options) *Mangareader { } } -func (m *Mangareader) retrieveImageLinks(comic *core.ComicIssue) ([]string, error) { - var links []string +func (m *Mangareader) requestContext() (context.Context, context.CancelFunc) { + return context.WithTimeout(context.Background(), m.options.RequestTimeout) +} - response, err := soup.Get(comic.Source.URL) +func (m *Mangareader) retrieveImageLinks(comic *core.ComicIssue) ([]string, error) { + ctx, cancel := m.requestContext() + defer cancel() + response, err := fetchHTML(ctx, m.options.Client, comic.Source.URL) if err != nil { return nil, err } + var links []string doc := soup.HTMLParse(response) for _, t := range doc.FindAll("img") { imageURL := t.Attrs()["data-src"] @@ -53,7 +59,10 @@ func (m *Mangareader) isSingleIssue(url string) bool { func (m *Mangareader) retrieveLastIssue(url string) (string, error) { url = strings.Join(util.TrimAndSplitURL(url)[:4], "/") - response, err := soup.Get(url) + ctx, cancel := m.requestContext() + defer cancel() + + response, err := fetchHTML(ctx, m.options.Client, url) if err != nil { return "", err } @@ -80,13 +89,15 @@ func (m *Mangareader) RetrieveIssueLinks() ([]string, error) { return []string{url}, nil } - var links []string + ctx, cancel := m.requestContext() + defer cancel() - response, err := soup.Get(url) + response, err := fetchHTML(ctx, m.options.Client, url) if err != nil { return nil, err } + var links []string doc := soup.HTMLParse(response) nodes := doc.Find("table", "class", "d48").FindAll("tr") for _, node := range nodes { diff --git a/pkg/sites/mangareader_test.go b/pkg/sites/mangareader_test.go index 75af022c..ffe46889 100644 --- a/pkg/sites/mangareader_test.go +++ b/pkg/sites/mangareader_test.go @@ -60,8 +60,9 @@ func TestMangareaderScraper(t *testing.T) { defer server.Close() opts := &config.Options{ - URL: server.URL + mangareaderIssuePath, - Logger: logger.NewLogger(false, nil), + URL: server.URL + mangareaderIssuePath, + Logger: logger.NewLogger(false, nil), + RequestTimeout: config.DefaulltRequestTimeout, } scraper := NewMangareader(opts) diff --git a/pkg/sites/mangatown.go b/pkg/sites/mangatown.go index b9b40cc3..baf4c6bb 100644 --- a/pkg/sites/mangatown.go +++ b/pkg/sites/mangatown.go @@ -1,6 +1,7 @@ package sites import ( + "context" "fmt" "strings" @@ -22,6 +23,10 @@ func NewMangatown(options *config.Options) *Mangatown { } } +func (m *Mangatown) requestContext() (context.Context, context.CancelFunc) { + return context.WithTimeout(context.Background(), m.options.RequestTimeout) +} + func (m *Mangatown) findPages(document *soup.Root) []string { var pages []string @@ -37,11 +42,10 @@ func (m *Mangatown) findPages(document *soup.Root) []string { } func (m *Mangatown) retrieveImageLinks(comic *core.ComicIssue) ([]string, error) { - var links []string - var link string - - response, err := soup.Get(comic.Source.URL) + ctx, cancel := m.requestContext() + defer cancel() + response, err := fetchHTML(ctx, m.options.Client, comic.Source.URL) if err != nil { return nil, err } @@ -49,10 +53,14 @@ func (m *Mangatown) retrieveImageLinks(comic *core.ComicIssue) ([]string, error) document := soup.HTMLParse(response) pages := m.findPages(&document) + var links []string + var link string for _, page := range pages { link = fmt.Sprintf("%s%s.html", comic.Source.URL, page) - response, err := soup.Get(link) + ctx, cancel := m.requestContext() + defer cancel() + response, err := fetchHTML(ctx, m.options.Client, link) if err != nil { return nil, err } @@ -76,8 +84,10 @@ func (m *Mangatown) isSingleIssue(url string) bool { func (m *Mangatown) retrieveLastIssue(url string) (string, error) { url = strings.Join(util.TrimAndSplitURL(url)[:5], "/") - response, err := soup.Get(url) + ctx, cancel := m.requestContext() + defer cancel() + response, err := fetchHTML(ctx, m.options.Client, url) if err != nil { return "", err } @@ -105,9 +115,10 @@ func (m *Mangatown) RetrieveIssueLinks() ([]string, error) { return []string{url}, nil } - var links []string + ctx, cancel := m.requestContext() + defer cancel() - response, err := soup.Get(url) + response, err := fetchHTML(ctx, m.options.Client, url) if err != nil { return nil, err } @@ -115,6 +126,7 @@ func (m *Mangatown) RetrieveIssueLinks() ([]string, error) { doc := soup.HTMLParse(response) chapters := doc.Find("ul", "class", "chapter_list").FindAll("a") + var links []string for _, chapter := range chapters { url := "https://mangatown.com" + chapter.Attrs()["href"] if util.IsURLValid(url) { diff --git a/pkg/sites/mangatown_test.go b/pkg/sites/mangatown_test.go index 7d9772f6..03abe389 100644 --- a/pkg/sites/mangatown_test.go +++ b/pkg/sites/mangatown_test.go @@ -73,8 +73,9 @@ func TestMangatownScraper(t *testing.T) { defer server.Close() opts := &config.Options{ - URL: server.URL + mangatownIssuePath, - Logger: logger.NewLogger(false, nil), + URL: server.URL + mangatownIssuePath, + Logger: logger.NewLogger(false, nil), + RequestTimeout: config.DefaulltRequestTimeout, } scraper := NewMangatown(opts) diff --git a/pkg/sites/readallcomics.go b/pkg/sites/readallcomics.go index 096d2e29..649dca38 100644 --- a/pkg/sites/readallcomics.go +++ b/pkg/sites/readallcomics.go @@ -1,6 +1,7 @@ package sites import ( + "context" "fmt" "strings" @@ -24,16 +25,22 @@ func NewReadallcomics(options *config.Options) *Readallcomics { } } +func (r *Readallcomics) requestContext() (context.Context, context.CancelFunc) { + return context.WithTimeout(context.Background(), r.options.RequestTimeout) +} + func (r *Readallcomics) retrieveImageLinks(comic *core.ComicIssue) ([]string, error) { - var links []string + ctx, cancel := r.requestContext() + defer cancel() - response, err := soup.Get(comic.Source.URL) + response, err := fetchHTML(ctx, r.options.Client, comic.Source.URL) if err != nil { - return links, err + return nil, err } document := soup.HTMLParse(response) + var links []string images := document.FindAll("img") for _, img := range images { src, ok := img.Attrs()["src"] @@ -54,15 +61,18 @@ func (r *Readallcomics) retrieveImageLinks(comic *core.ComicIssue) ([]string, er // Retrieve issues links from main comic page or from comic issue. func (r *Readallcomics) getIssues(url string) ([]string, error) { - var links []string - response, err := soup.Get(url) + ctx, cancel := r.requestContext() + defer cancel() + + response, err := fetchHTML(ctx, r.options.Client, url) if err != nil { return nil, err } doc := soup.HTMLParse(response) + var links []string if strings.Contains(url, "category") { chapterList := doc.Find("ul", "class", "list-story") if chapterList.Error != nil { diff --git a/pkg/sites/readallcomics_test.go b/pkg/sites/readallcomics_test.go index 39174636..710b4704 100644 --- a/pkg/sites/readallcomics_test.go +++ b/pkg/sites/readallcomics_test.go @@ -56,8 +56,9 @@ func TestReadAllComicsScraper(t *testing.T) { defer server.Close() opts := &config.Options{ - URL: server.URL + readAllIssuePath, - Logger: logger.NewLogger(false, nil), + URL: server.URL + readAllIssuePath, + Logger: logger.NewLogger(false, nil), + RequestTimeout: config.DefaulltRequestTimeout, } scraper := NewReadallcomics(opts) diff --git a/pkg/sites/readcomiconline.go b/pkg/sites/readcomiconline.go index 484473f1..8535ff81 100644 --- a/pkg/sites/readcomiconline.go +++ b/pkg/sites/readcomiconline.go @@ -1,6 +1,7 @@ package sites import ( + "context" "encoding/base64" "fmt" "regexp" @@ -9,7 +10,6 @@ import ( "github.com/Girbons/comics-downloader/pkg/config" "github.com/Girbons/comics-downloader/pkg/core" "github.com/Girbons/comics-downloader/pkg/util" - "github.com/anaskhan96/soup" ) var baseUrl = "https://readcomiconline.li" @@ -26,6 +26,10 @@ func NewReadComiconline(options *config.Options) *ReadComicOnline { } } +func (c *ReadComicOnline) requestContext() (context.Context, context.CancelFunc) { + return context.WithTimeout(context.Background(), c.options.RequestTimeout) +} + func deobfuscateUrl(imageLink string) (string, error) { imageLink = strings.ReplaceAll(imageLink, "_x236", "d") imageLink = strings.ReplaceAll(imageLink, "_x945", "g") @@ -72,7 +76,10 @@ func (c *ReadComicOnline) retrieveImageLinks(comic *core.ComicIssue) ([]string, c.options.Logger.Debugf("readcomiconline: fetching %s", fetchURL) } - response, err := soup.Get(fetchURL) + ctx, cancel := c.requestContext() + defer cancel() + + response, err := fetchHTML(ctx, c.options.Client, fetchURL) if err != nil { if c.options.Logger != nil { c.options.Logger.Errorf("readcomiconline: request to %s failed: %v", fetchURL, err) @@ -126,9 +133,10 @@ func (c *ReadComicOnline) isSingleIssue(url string) bool { } func (c *ReadComicOnline) retrieveLastIssue(url string) (string, error) { - var lastIssue string + ctx, cancel := c.requestContext() + defer cancel() - response, err := soup.Get(url) + response, err := fetchHTML(ctx, c.options.Client, url) if err != nil { return "", err } @@ -136,7 +144,7 @@ func (c *ReadComicOnline) retrieveLastIssue(url string) (string, error) { name := util.TrimAndSplitURL(url)[4] re := regexp.MustCompile("]+href=\"([^\">]+" + "/" + name + "/.+)\"") match := re.FindAllStringSubmatch(response, -1) - lastIssue = baseUrl + strings.Split(match[0][1], "?")[0] + lastIssue := baseUrl + strings.Split(match[0][1], "?")[0] return lastIssue, nil } @@ -162,7 +170,10 @@ func (c *ReadComicOnline) RetrieveIssueLinks() ([]string, error) { links []string ) - response, err := soup.Get(url) + ctx, cancel := c.requestContext() + defer cancel() + + response, err := fetchHTML(ctx, c.options.Client, url) if err != nil { return nil, err } diff --git a/pkg/sites/readcomiconline_test.go b/pkg/sites/readcomiconline_test.go index bbdec83f..a95cdb87 100644 --- a/pkg/sites/readcomiconline_test.go +++ b/pkg/sites/readcomiconline_test.go @@ -55,8 +55,9 @@ func TestReadComicOnlineScraper(t *testing.T) { defer func() { baseUrl = originalBase }() opts := &config.Options{ - URL: server.URL + rcoIssuePath, - Logger: logger.NewLogger(false, nil), + URL: server.URL + rcoIssuePath, + Logger: logger.NewLogger(false, nil), + RequestTimeout: config.DefaulltRequestTimeout, } scraper := NewReadComiconline(opts) From a3346b0162b41fdc3e0a641762dcf08ae89a22eb Mon Sep 17 00:00:00 2001 From: ProjectDislocate <173194962+ProjectDislocate@users.noreply.github.com> Date: Fri, 6 Mar 2026 17:54:31 -0500 Subject: [PATCH 24/79] feat: add author, lang, and desc to epubs --- pkg/core/core.go | 27 +++++++++++++++++++-- pkg/core/core_test.go | 14 +++++------ pkg/core/metadata.go | 47 ++++++++++++++++++++++++++----------- pkg/core/metadata_test.go | 49 +++++++++++++++++++++++++++++++++++++++ pkg/core/output.go | 10 +++----- 5 files changed, 118 insertions(+), 29 deletions(-) create mode 100644 pkg/core/metadata_test.go diff --git a/pkg/core/core.go b/pkg/core/core.go index 8261ce17..3411db29 100644 --- a/pkg/core/core.go +++ b/pkg/core/core.go @@ -53,8 +53,31 @@ func (comic *ComicIssue) makeEPUB(options *config.Options, images *DownloadResul e := epub.NewEpub(comic.IssueNumber) e.SetTitle(fmt.Sprintf("%s-%s", comic.Name, comic.IssueNumber)) - if comic.Author != "" { - e.SetAuthor(comic.Author) + if len(comic.SeriesMetadata.Creators) > 0 { + setAuthor := false + writerRole := string(CreatorRoleWriter) + for _, creator := range comic.SeriesMetadata.Creators { + if strings.EqualFold(string(creator.Role), writerRole) { + e.SetAuthor(creator.Name) + setAuthor = true + break + } + } + + if !setAuthor { + // if no creator with role "Writer" is found, set the author to the first creator in the list + author := comic.SeriesMetadata.Creators[0].Name + e.SetAuthor(author) + } + } + + if comic.LanguageISO != nil { + e.SetLang(*comic.LanguageISO) + } + + comicDescription := comic.getDescriptionForLanguage(options, options.Country) + if comicDescription != "" { + e.SetDescription(comicDescription) } for _, file := range images.FilePaths { diff --git a/pkg/core/core_test.go b/pkg/core/core_test.go index aff1324b..3ab308f7 100644 --- a/pkg/core/core_test.go +++ b/pkg/core/core_test.go @@ -117,13 +117,13 @@ func TestMakeComicEPUB(t *testing.T) { opts := newTestOptions(t, server) comic := &ComicIssue{ - Name: "bar", - Source: &ComicSource{Name: "test-source", URL: server.URL}, - IssueNumber: "42", - Author: "Author", - OutputFormat: EPUB, - ImagesFormat: "png", - ImageLinks: buildLinks(server, 2), + Name: "bar", + Source: &ComicSource{Name: "test-source", URL: server.URL}, + IssueNumber: "42", + OutputFormat: EPUB, + ImagesFormat: "png", + ImageLinks: buildLinks(server, 2), + SeriesMetadata: &SeriesMetadata{}, } require.NoError(t, comic.MakeComic(opts)) diff --git a/pkg/core/metadata.go b/pkg/core/metadata.go index ee5176e6..9f491265 100644 --- a/pkg/core/metadata.go +++ b/pkg/core/metadata.go @@ -1,7 +1,10 @@ package core import ( + "strings" "time" + + "github.com/Girbons/comics-downloader/pkg/config" ) type AgeRating string @@ -30,11 +33,11 @@ type ComicSource struct { URL string // URL of the comic/manga issue } -type CreatorsRole string +type CreatorRole string type SeriesCreator struct { Name string - Role CreatorsRole + Role CreatorRole } type SeriesMetadata struct { @@ -56,8 +59,7 @@ type SeriesMetadata struct { // ComicIssue struct contains all the informations about a comic type ComicIssue struct { - Author string // Remove in favor of SeriesMetadata.Creators?? - Name string // Issue name/title + Name string // Issue name/title IssueNumber string Volume *string @@ -73,15 +75,15 @@ type ComicIssue struct { } const ( - CreatorRoleUnknown = "Unknown" - CreatorRoleWriter = "Writer" - CreatorRolePenciller = "Penciller" - CreatorRoleInker = "Inker" - CreatorRoleColorist = "Colorist" - CreatorRoleLetterer = "Letterer" - CreatorRoleCoverArtist = "CoverArtist" - CreatorRoleEditor = "Editor" - CreatorRoleTranslator = "Translator" + CreatorRoleUnknown CreatorRole = "Unknown" + CreatorRoleWriter CreatorRole = "Writer" + CreatorRolePenciller CreatorRole = "Penciller" + CreatorRoleInker CreatorRole = "Inker" + CreatorRoleColorist CreatorRole = "Colorist" + CreatorRoleLetterer CreatorRole = "Letterer" + CreatorRoleCoverArtist CreatorRole = "CoverArtist" + CreatorRoleEditor CreatorRole = "Editor" + CreatorRoleTranslator CreatorRole = "Translator" ) // func SourceAuthorRoleToSeriesAuthorRole(sourceRole string) string { @@ -107,3 +109,22 @@ const ( // return CreatorRoleUnknown // } // } + +func (c *ComicIssue) getDescriptionForLanguage(options *config.Options, lang string) string { + if len(c.SeriesMetadata.Description) <= 0 { + return "" + } + + for lang, desc := range c.SeriesMetadata.Description { + if options.Country == "" || options.Country == strings.ToLower(lang) { + return desc + } + } + + // if no description matching the country option is found, set the description to the first one in the map + for _, desc := range c.SeriesMetadata.Description { + return desc + } + + return "" +} diff --git a/pkg/core/metadata_test.go b/pkg/core/metadata_test.go new file mode 100644 index 00000000..0f4442fb --- /dev/null +++ b/pkg/core/metadata_test.go @@ -0,0 +1,49 @@ +package core + +import ( + "testing" + + "github.com/Girbons/comics-downloader/pkg/config" + "github.com/stretchr/testify/require" +) + +func TestGetDescriptionForLanguage_MatchingLanguage(t *testing.T) { + opts := &config.Options{Country: "en"} + c := &ComicIssue{ + SeriesMetadata: &SeriesMetadata{ + Description: map[string]string{ + "en": "English description", + "jp": "Japanese description", + }, + }, + } + + desc := c.getDescriptionForLanguage(opts, "en") + require.Equal(t, "English description", desc) +} + +func TestGetDescriptionForLanguage_FallbackSingleEntry(t *testing.T) { + opts := &config.Options{Country: "fr"} + c := &ComicIssue{ + SeriesMetadata: &SeriesMetadata{ + Description: map[string]string{ + "es": "Spanish description", + }, + }, + } + + desc := c.getDescriptionForLanguage(opts, "fr") + require.Equal(t, "Spanish description", desc) +} + +func TestGetDescriptionForLanguage_Empty(t *testing.T) { + opts := &config.Options{Country: "en"} + c := &ComicIssue{ + SeriesMetadata: &SeriesMetadata{ + Description: map[string]string{}, + }, + } + + desc := c.getDescriptionForLanguage(opts, "en") + require.Equal(t, "", desc) +} diff --git a/pkg/core/output.go b/pkg/core/output.go index 2997eeab..13b02f22 100644 --- a/pkg/core/output.go +++ b/pkg/core/output.go @@ -86,13 +86,9 @@ func (comic *ComicIssue) makeComicInfoXML(options *config.Options, images *Downl break } } - for lang, localizedDesc := range comic.SeriesMetadata.Description { - // TODO: use the country option to select the localized description instead of defaulting to English, or add a separate option for the localized description language - if lang == "en" { - comicInfo.CreateElement("Summary").SetText(localizedDesc) - break - } - } + + comicDescription := comic.getDescriptionForLanguage(options, options.Country) + comicInfo.CreateElement("Summary").SetText(comicDescription) comicInfo.CreateElement("Number").SetText(comic.IssueNumber) if comic.Volume != nil { From 94f549afd44c36c4ee2e0cf3d4d27f30727d87b2 Mon Sep 17 00:00:00 2001 From: ProjectDislocate <173194962+ProjectDislocate@users.noreply.github.com> Date: Fri, 6 Mar 2026 20:25:11 -0500 Subject: [PATCH 25/79] feat: improve mangadex metadata and use locale title for epub --- pkg/core/core.go | 9 +- pkg/core/metadata.go | 27 ++++- pkg/core/metadata_test.go | 6 +- pkg/core/output.go | 13 +-- pkg/sites/mangadex.go | 211 +++++++++++++++++++++++++++++++++---- pkg/sites/mangadex_test.go | 145 +++++++++++++++++++++++-- 6 files changed, 365 insertions(+), 46 deletions(-) diff --git a/pkg/core/core.go b/pkg/core/core.go index 3411db29..088a9108 100644 --- a/pkg/core/core.go +++ b/pkg/core/core.go @@ -49,9 +49,10 @@ func ensureClient(options *config.Options) *httpclient.ComicClient { // makeEPUB creates the epub file. func (comic *ComicIssue) makeEPUB(options *config.Options, images *DownloadResult) error { isCoverSet := false - imgTag := `Cover Image` - e := epub.NewEpub(comic.IssueNumber) - e.SetTitle(fmt.Sprintf("%s-%s", comic.Name, comic.IssueNumber)) + imgTag := `Comic Page` + + localizedTitle := comic.getLocalizedTitle(options) + e := epub.NewEpub(localizedTitle) if len(comic.SeriesMetadata.Creators) > 0 { setAuthor := false @@ -75,7 +76,7 @@ func (comic *ComicIssue) makeEPUB(options *config.Options, images *DownloadResul e.SetLang(*comic.LanguageISO) } - comicDescription := comic.getDescriptionForLanguage(options, options.Country) + comicDescription := comic.getLocalizedDescription(options) if comicDescription != "" { e.SetDescription(comicDescription) } diff --git a/pkg/core/metadata.go b/pkg/core/metadata.go index 9f491265..fed61870 100644 --- a/pkg/core/metadata.go +++ b/pkg/core/metadata.go @@ -110,7 +110,7 @@ const ( // } // } -func (c *ComicIssue) getDescriptionForLanguage(options *config.Options, lang string) string { +func (c *ComicIssue) getLocalizedDescription(options *config.Options) string { if len(c.SeriesMetadata.Description) <= 0 { return "" } @@ -128,3 +128,28 @@ func (c *ComicIssue) getDescriptionForLanguage(options *config.Options, lang str return "" } + +func (c *ComicIssue) getLocalizedTitle(options *config.Options) string { + if len(c.SeriesMetadata.LocalizedTitle) <= 0 { + // fmt.Println("No localized titles") + return c.SeriesMetadata.Title + } + + // for key, value := range c.SeriesMetadata.LocalizedTitle { + // options.Logger.Infof("%s: %s\n", key, value) + // } + + for lang, title := range c.SeriesMetadata.LocalizedTitle { + if options.Country == "" || options.Country == strings.ToLower(lang) { + // options.Logger.Infof("Found desired title \"%s\" in the lang %s\n", title, lang) + return title + } + } + + // if no title matching the country option is found, set the title to the first one in the map + for _, title := range c.SeriesMetadata.LocalizedTitle { + return title + } + + return c.SeriesMetadata.Title +} diff --git a/pkg/core/metadata_test.go b/pkg/core/metadata_test.go index 0f4442fb..89318564 100644 --- a/pkg/core/metadata_test.go +++ b/pkg/core/metadata_test.go @@ -18,7 +18,7 @@ func TestGetDescriptionForLanguage_MatchingLanguage(t *testing.T) { }, } - desc := c.getDescriptionForLanguage(opts, "en") + desc := c.getLocalizedDescription(opts) require.Equal(t, "English description", desc) } @@ -32,7 +32,7 @@ func TestGetDescriptionForLanguage_FallbackSingleEntry(t *testing.T) { }, } - desc := c.getDescriptionForLanguage(opts, "fr") + desc := c.getLocalizedDescription(opts) require.Equal(t, "Spanish description", desc) } @@ -44,6 +44,6 @@ func TestGetDescriptionForLanguage_Empty(t *testing.T) { }, } - desc := c.getDescriptionForLanguage(opts, "en") + desc := c.getLocalizedDescription(opts) require.Equal(t, "", desc) } diff --git a/pkg/core/output.go b/pkg/core/output.go index 13b02f22..e74cd00a 100644 --- a/pkg/core/output.go +++ b/pkg/core/output.go @@ -78,16 +78,11 @@ func (comic *ComicIssue) makeComicInfoXML(options *config.Options, images *Downl comicInfo.CreateElement("Series").SetText(comic.SeriesMetadata.Title) comicInfo.CreateElement("Title").SetText(comic.Name) - for lang, localizedTitle := range comic.SeriesMetadata.LocalizedTitle { - // TODO: use the country option to select the localized title instead of defaulting to English, or add a separate option for the localized title language - if lang == "en" { - // non-standard field - comicInfo.CreateElement("LocalizedSeries").SetText(localizedTitle) - break - } - } - comicDescription := comic.getDescriptionForLanguage(options, options.Country) + localizedTitle := comic.getLocalizedTitle(options) + comicInfo.CreateElement("LocalizedSeries").SetText(localizedTitle) + + comicDescription := comic.getLocalizedDescription(options) comicInfo.CreateElement("Summary").SetText(comicDescription) comicInfo.CreateElement("Number").SetText(comic.IssueNumber) diff --git a/pkg/sites/mangadex.go b/pkg/sites/mangadex.go index 1e0e6bee..cd3d5fb6 100644 --- a/pkg/sites/mangadex.go +++ b/pkg/sites/mangadex.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "net/url" + "strconv" "strings" "time" @@ -51,7 +52,24 @@ func joinURL(base, suffix string) string { return strings.TrimRight(base, "/") + "/" + strings.TrimLeft(suffix, "/") } -func (m *Mangadex) getManga(mangaID string) (string, error) { +type mangadexSeries struct { + ID string + + Title string // the title we want to use for the series, which may be localized based on the country option + LocalizedTitle map[string]string + Description map[string]string + IsManga bool + + OriginalLanguage string + Year *int + + ContentRating core.AgeRating + Tags []string + Genres []string + WebLinks []string +} + +func (m *Mangadex) getMangaInfo(mangaID string) (mangadexSeries, error) { ctx, cancel := m.requestContext() defer cancel() @@ -60,32 +78,163 @@ func (m *Mangadex) getManga(mangaID string) (string, error) { Result string `json:"result"` Data struct { Attributes struct { - Titles map[string]string `json:"title"` + Title map[string]string `json:"title"` + AltTitles []map[string]string `json:"altTitles"` + Description map[string]string `json:"description"` + Links map[string]string `json:"links"` + OriginalLanguage string `json:"originalLanguage"` + Year *int `json:"year"` + ContentRating string `json:"contentRating"` + PublicationDemographic *string `json:"publicationDemographic"` + Tags []struct { + ID string `json:"id"` + Type string `json:"type"` // should be "tag" + Attributes struct { + Name map[string]string `json:"name"` + Description map[string]string `json:"description"` + Group string `json:"group"` + Version int `json:"version"` + } `json:"attributes"` + } `json:"tags"` + // AvailableTranslatedLanguages []string `json:"availableTranslatedLanguages"` } `json:"attributes"` } `json:"data"` } if err := fetchJSON(ctx, m.client, endpoint, &mangaRes); err != nil { - return "", err + return mangadexSeries{}, err } if strings.ToLower(mangaRes.Result) != "ok" { - return "", fmt.Errorf("unexpected response") + return mangadexSeries{}, fmt.Errorf("unexpected response") } - // TODO: set localized title based on country option instead of setting main title to that language - // then need to update how paths are generated to use the localized title instead of the main title if the country option is set - for lang, title := range mangaRes.Data.Attributes.Titles { + manga := mangadexSeries{ + LocalizedTitle: mangaRes.Data.Attributes.Title, + Description: mangaRes.Data.Attributes.Description, + IsManga: true, // default to true since it's a manga site + } + + // Set titles + foundTitle := false + for lang, title := range mangaRes.Data.Attributes.Title { if m.country == "" || m.country == strings.ToLower(lang) { - return title, nil + // TODO: set localized title based on country option instead of main title + // need to update how paths are generated to use the localized title instead of the main title if the country option is set + manga.Title = title + foundTitle = true + break + } + + // manga.LocalizedTitle[lang] = title + } + + // try and fill in any missing localized titles + for _, grouping := range mangaRes.Data.Attributes.AltTitles { + for lang, title := range grouping { + if _, exists := manga.LocalizedTitle[lang]; !exists { + manga.LocalizedTitle[lang] = title + } + + // if main title is missing, try to use the alt titles to fill it + if !foundTitle && (m.country == "" || m.country == strings.ToLower(lang)) { + manga.Title = title + foundTitle = true + } + } + } + + if !foundTitle { + // If still haven't found anything fallback to any available title. + for _, title := range mangaRes.Data.Attributes.Title { + manga.Title = title + } + } + + // handle tags and genres + if mangaRes.Data.Attributes.PublicationDemographic != nil { + manga.Tags = append(manga.Tags, *mangaRes.Data.Attributes.PublicationDemographic) + } + for _, tag := range mangaRes.Data.Attributes.Tags { + // using English name for tags since they usally don't have localized names + name, ok := tag.Attributes.Name["en"] + if !ok { + // if no English name, try to use the first available name + for _, n := range tag.Attributes.Name { + name = n + break + } + } + + if name != "" { + continue + } + + switch tag.Attributes.Group { + case "genre": + manga.Genres = append(manga.Genres, name) + case "tag", "format": + manga.Tags = append(manga.Tags, name) } } - // Fallback to any available title. - for _, title := range mangaRes.Data.Attributes.Titles { - return title, nil + // see https://api.mangadex.org/docs/3-enumerations/#manga-content-rating + switch mangaRes.Data.Attributes.ContentRating { + case "safe": + manga.ContentRating = core.AgeRatingEveryone + case "suggestive", "erotica": + manga.ContentRating = core.AgeRatingMature + case "pornographic": + manga.ContentRating = core.AgeRatingAO18 + default: + manga.ContentRating = core.AgeRatingEveryone + } + + for key, link := range mangaRes.Data.Attributes.Links { + fullURL := m.mangaLinkToFullURL(key, link) + if fullURL != "" { + manga.WebLinks = append(manga.WebLinks, fullURL) + } } + manga.OriginalLanguage = mangaRes.Data.Attributes.OriginalLanguage + manga.Year = mangaRes.Data.Attributes.Year - return "", fmt.Errorf("no title found for manga %s", mangaID) + return manga, nil +} + +func (m *Mangadex) mangaLinkToFullURL(key, link string) string { + switch key { + case "al": + return "https://www.anilist.co/manga/" + link + case "ap": + return "https://www.animeplanet.com/manga/" + link + case "bw": + return "https://www.bookwalker.jp/" + link + case "mu": + return "https://www.mangaupdates.com/series.html?id=" + link + case "nu": + return "https://www.novelupdates.com/series/" + link + case "kt": + // if int use id + if _, err := strconv.Atoi(link); err == nil { + return "https://kitsu.io/api/edge/manga/" + link + } + // else use slug + return "https://kitsu.io/api/edge/manga?filter[slug]=" + link + case "amz": + return link + case "ebj": + return link + case "mal": + return "https://myanimelist.net/manga/" + link + case "cdj": + return link + case "raw": + return link + case "engtl": + return link + default: + return "" + } } // getChapters fetches chapter URLs for the given manga. @@ -148,8 +297,8 @@ type mangadexChapter struct { ImageLinks []string } -// getChapter retrieves metadata and image links for a single chapter. -func (m *Mangadex) getChapter(chapterID string) (chapterInfo mangadexChapter, err error) { +// getChapterInfo retrieves metadata and image links for a single chapter. +func (m *Mangadex) getChapterInfo(chapterID string) (chapterInfo mangadexChapter, err error) { ctx, cancel := m.requestContext() defer cancel() @@ -269,7 +418,7 @@ func (m *Mangadex) GetInfo(urlValue string) (string, string) { } switch parts[3] { case "chapter": - chapter, err := m.getChapter(parts[4]) + chapter, err := m.getChapterInfo(parts[4]) if err != nil { return "", "" } @@ -285,18 +434,18 @@ func (m *Mangadex) GetInfo(urlValue string) (string, string) { if chapter.ChapterTitle != "" { chapterTitle += fmt.Sprintf(", %s", chapter.ChapterTitle) } - mangaTitle, err := m.getManga(chapter.MangaID) + manga, err := m.getMangaInfo(chapter.MangaID) if err != nil { return "", chapterTitle } - return mangaTitle, chapterTitle + return manga.Title, chapterTitle case "title": - mangaTitle, err := m.getManga(parts[4]) + manga, err := m.getMangaInfo(parts[4]) if err != nil { return "", "" } - return mangaTitle, "" + return manga.Title, "" default: return "", "" } @@ -304,11 +453,21 @@ func (m *Mangadex) GetInfo(urlValue string) (string, string) { // Initialize loads links and metadata from mangadex. func (m *Mangadex) Initialize(comic *core.ComicIssue) error { + if comic == nil { + return fmt.Errorf("comic is nil") + } + if comic.Source == nil { + return fmt.Errorf("comic source is nil") + } parts := util.TrimAndSplitURL(comic.Source.URL) if len(parts) < 5 { return fmt.Errorf("URL not supported") } - chapter, err := m.getChapter(parts[4]) + chapter, err := m.getChapterInfo(parts[4]) + if err != nil { + return err + } + manga, err := m.getMangaInfo(chapter.MangaID) if err != nil { return err } @@ -319,6 +478,18 @@ func (m *Mangadex) Initialize(comic *core.ComicIssue) error { comic.LanguageISO = &chapter.TranslatedLanguage comic.ReleaseDate = &chapter.PublishAt + if comic.SeriesMetadata == nil { + comic.SeriesMetadata = &core.SeriesMetadata{} + } + + comic.SeriesMetadata.AgeRating = &manga.ContentRating + comic.SeriesMetadata.Description = manga.Description + comic.SeriesMetadata.Genres = manga.Genres + comic.SeriesMetadata.Tags = manga.Tags + comic.SeriesMetadata.Title = manga.Title + comic.SeriesMetadata.WebLinks = manga.WebLinks + comic.SeriesMetadata.IsManga = &manga.IsManga + comic.ImageLinks = chapter.ImageLinks return nil diff --git a/pkg/sites/mangadex_test.go b/pkg/sites/mangadex_test.go index 6f2207ec..361818e8 100644 --- a/pkg/sites/mangadex_test.go +++ b/pkg/sites/mangadex_test.go @@ -31,11 +31,52 @@ func setupMangadexServer() *httptest.Server { }`) case strings.HasPrefix(r.URL.Path, "/manga/series-1"): w.Header().Set("Content-Type", "application/json") + // TODO: add another series test where year, demographic, tags, etc are all missing. want to test minimum viable response fmt.Fprint(w, `{ "result":"ok", "data":{ "attributes":{ - "title":{"en":"Test Manga","jp":"テスト"} + "title":{"jp":"テスト"}, + "altTitles":[ + {"en":"Test Manga"}, + {"fr":"Manga de Test"}, + {"zh": "测试漫画"} + ], + "description":{"en":"Test manga description"}, + "publicationDemographic":"shounen", + "contentRating":"safe", + "tags":[ + { + "id":"tag-1", + "type":"tag", + "attributes":{ + "name":{"en":"Romance"}, + "group":"genre", + "version":1 + } + }, + { + "id":"tag-2", + "type":"tag", + "attributes":{ + "name":{"en":"School Life"}, + "group":"theme", + "version":1 + } + }, + { + "id":"tag-3", + "type":"tag", + "attributes":{ + "name":{"en":"Doujinshi"}, + "group":"format", + "version":1 + } + } + ], + "links":{ + "al": "30642" + } } } }`) @@ -44,7 +85,13 @@ func setupMangadexServer() *httptest.Server { fmt.Fprint(w, `{ "result":"ok", "data":{ - "attributes":{"volume":"1","chapter":"1","title":"Start","publishAt":"2026-03-06T14:03:52.000Z","translatedLanguage":"en"}, + "attributes":{ + "volume":"1", + "chapter":"1", + "title":"Start", + "publishAt":"2026-03-06T14:03:52.000Z", + "translatedLanguage":"en" + }, "relationships":[{"id":"series-1","type":"manga"}] } }`) @@ -54,13 +101,60 @@ func setupMangadexServer() *httptest.Server { "result":"ok", "chapter":{"hash":"HASH","data":["001.png","002.png"]} }`) + case strings.HasPrefix(r.URL.Path, "/manga/series-2/aggregate"): + w.Header().Set("Content-Type", "application/json") + fmt.Fprint(w, `{ + "result":"ok", + "volumes":{ + "1":{ + "chapters":{ + "1":{"id":"chapter-1","chapter":"1"} + } + } + } + }`) + case strings.HasPrefix(r.URL.Path, "/manga/series-2"): + w.Header().Set("Content-Type", "application/json") + // TODO: add another series test where year, demographic, tags, etc are all missing. want to test minimum viable response + fmt.Fprint(w, `{ + "result":"ok", + "data":{ + "attributes":{ + "title":{"jp":"テスト"}, + "altTitles":[ + {"en":"Test Manga"} + ], + } + } + }`) + case strings.HasPrefix(r.URL.Path, "/chapter/chapter-2"): + w.Header().Set("Content-Type", "application/json") + fmt.Fprint(w, `{ + "result":"ok", + "data":{ + "attributes":{ + "volume":"1", + "chapter":"1", + "title":"Start", + "publishAt":"2026-03-06T14:03:52.000Z", + "translatedLanguage":"en" + }, + "relationships":[{"id":"series-2","type":"manga"}] + } + }`) + case strings.HasPrefix(r.URL.Path, "/at-home/server/chapter-2"): + w.Header().Set("Content-Type", "application/json") + fmt.Fprint(w, `{ + "result":"ok", + "chapter":{"hash":"HASH","data":["001.png","002.png"]} + }`) default: http.NotFound(w, r) } })) } -func newTestMangadex(t *testing.T) (*Mangadex, func()) { +func newTestMangadex(t *testing.T, series, lang string) (*Mangadex, func()) { t.Helper() server := setupMangadexServer() @@ -71,8 +165,8 @@ func newTestMangadex(t *testing.T) (*Mangadex, func()) { ) opts := &config.Options{ - URL: server.URL + "/title/series-1/naruto", - Country: "en", + URL: server.URL + fmt.Sprintf("/title/%s/naruto", series), + Country: lang, SourceName: "mangadex.org", Logger: logger.NewLogger(false, nil), Client: client, @@ -92,7 +186,7 @@ func newTestMangadex(t *testing.T) (*Mangadex, func()) { } func TestMangadexRetrieveIssueLinks(t *testing.T) { - md, cleanup := newTestMangadex(t) + md, cleanup := newTestMangadex(t, "series-1", "en") defer cleanup() md.options.All = true @@ -103,7 +197,7 @@ func TestMangadexRetrieveIssueLinks(t *testing.T) { } func TestMangadexInitialize(t *testing.T) { - md, cleanup := newTestMangadex(t) + md, cleanup := newTestMangadex(t, "series-1", "en") defer cleanup() comic := &core.ComicIssue{ @@ -117,11 +211,44 @@ func TestMangadexInitialize(t *testing.T) { }, comic.ImageLinks) } -func TestMangadexGetInfo(t *testing.T) { - md, cleanup := newTestMangadex(t) +func TestMangadexInitializeMinimum(t *testing.T) { + md, cleanup := newTestMangadex(t, "series-2", "en") + defer cleanup() + + comic := &core.ComicIssue{ + Source: &core.ComicSource{Name: "test-source", URL: md.chapterBase + "/chapter-1"}, + } + err := md.Initialize(comic) + require.NoError(t, err) + require.Equal(t, []string{ + md.uploadsBase + "/HASH/001.png", + md.uploadsBase + "/HASH/002.png", + }, comic.ImageLinks) +} + +func TestMangadexGetInfoEnglish(t *testing.T) { + md, cleanup := newTestMangadex(t, "series-1", "en") defer cleanup() title, chapter := md.GetInfo(md.chapterBase + "/chapter-1") require.Equal(t, "Test Manga", title) require.Equal(t, "Vol 1 Chapter 1, Start", chapter) } + +func TestMangadexGetInfoJapanese(t *testing.T) { + md, cleanup := newTestMangadex(t, "series-1", "jp") + defer cleanup() + + title, chapter := md.GetInfo(md.chapterBase + "/chapter-1") + require.Equal(t, "テスト", title) + require.Equal(t, "Vol 1 Chapter 1, Start", chapter) +} + +func TestMangadexGetInfoNoCountry(t *testing.T) { + md, cleanup := newTestMangadex(t, "series-1", "") + defer cleanup() + + title, chapter := md.GetInfo(md.chapterBase + "/chapter-1") + require.Equal(t, "テスト", title) + require.Equal(t, "Vol 1 Chapter 1, Start", chapter) +} From f4e0d0b5aef6d0559de8b92ff2858c693f408eca Mon Sep 17 00:00:00 2001 From: ProjectDislocate <173194962+ProjectDislocate@users.noreply.github.com> Date: Fri, 6 Mar 2026 20:38:40 -0500 Subject: [PATCH 26/79] chore: cleanup todos --- pkg/core/output.go | 1 - pkg/sites/mangadex.go | 2 -- pkg/sites/mangadex_test.go | 2 -- 3 files changed, 5 deletions(-) diff --git a/pkg/core/output.go b/pkg/core/output.go index e74cd00a..96c1ef85 100644 --- a/pkg/core/output.go +++ b/pkg/core/output.go @@ -164,7 +164,6 @@ func (comic *ComicIssue) makeComicInfoXML(options *config.Options, images *Downl case CreatorRoleTranslator: translators = append(translators, creator.Name) } - // TODO: handle unknown roles } if len(writers) > 0 { comicInfo.CreateElement("Writer").SetText(strings.Join(writers, ",")) diff --git a/pkg/sites/mangadex.go b/pkg/sites/mangadex.go index cd3d5fb6..146e1a9d 100644 --- a/pkg/sites/mangadex.go +++ b/pkg/sites/mangadex.go @@ -118,8 +118,6 @@ func (m *Mangadex) getMangaInfo(mangaID string) (mangadexSeries, error) { foundTitle := false for lang, title := range mangaRes.Data.Attributes.Title { if m.country == "" || m.country == strings.ToLower(lang) { - // TODO: set localized title based on country option instead of main title - // need to update how paths are generated to use the localized title instead of the main title if the country option is set manga.Title = title foundTitle = true break diff --git a/pkg/sites/mangadex_test.go b/pkg/sites/mangadex_test.go index 361818e8..9221611e 100644 --- a/pkg/sites/mangadex_test.go +++ b/pkg/sites/mangadex_test.go @@ -31,7 +31,6 @@ func setupMangadexServer() *httptest.Server { }`) case strings.HasPrefix(r.URL.Path, "/manga/series-1"): w.Header().Set("Content-Type", "application/json") - // TODO: add another series test where year, demographic, tags, etc are all missing. want to test minimum viable response fmt.Fprint(w, `{ "result":"ok", "data":{ @@ -115,7 +114,6 @@ func setupMangadexServer() *httptest.Server { }`) case strings.HasPrefix(r.URL.Path, "/manga/series-2"): w.Header().Set("Content-Type", "application/json") - // TODO: add another series test where year, demographic, tags, etc are all missing. want to test minimum viable response fmt.Fprint(w, `{ "result":"ok", "data":{ From 407f0323e843967de446ebf6f5f8bc65ed7667a3 Mon Sep 17 00:00:00 2001 From: ProjectDislocate <173194962+ProjectDislocate@users.noreply.github.com> Date: Fri, 6 Mar 2026 20:41:47 -0500 Subject: [PATCH 27/79] chore: mark mangadex as working --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 109cbfbc..fe0f0649 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ - https://readcomiconline.li/ ⚠️ - https://www.mangareader.tv/ ⚠️ - https://www.mangatown.com/ ⚠️ -- https://mangadex.org/ ⚠️ +- https://mangadex.org/ - https://mangakakalot.com/ ⚠️ - https://manganato.com/ ⚠️ From 1e834495729feb9fb7dc2d6c41851c08be2766c4 Mon Sep 17 00:00:00 2001 From: ProjectDislocate <173194962+ProjectDislocate@users.noreply.github.com> Date: Fri, 6 Mar 2026 21:06:14 -0500 Subject: [PATCH 28/79] feat: GetInfo now returns an error instead of silently failing --- pkg/sites/base.go | 2 +- pkg/sites/comicextra.go | 4 ++-- pkg/sites/loader.go | 7 ++++++- pkg/sites/loader_test.go | 6 +++--- pkg/sites/mangadex.go | 16 ++++++++-------- pkg/sites/mangadex_test.go | 9 ++++++--- pkg/sites/mangakakalot.go | 7 +++---- pkg/sites/mangakakalot_test.go | 2 +- pkg/sites/manganato.go | 8 ++++---- pkg/sites/manganato_test.go | 3 ++- pkg/sites/mangareader.go | 10 +++++++--- pkg/sites/mangatown.go | 4 ++-- pkg/sites/readallcomics.go | 14 +++++++++----- pkg/sites/readallcomics_test.go | 3 ++- pkg/sites/readcomiconline.go | 4 ++-- 15 files changed, 58 insertions(+), 41 deletions(-) diff --git a/pkg/sites/base.go b/pkg/sites/base.go index eebfa94f..596f8605 100644 --- a/pkg/sites/base.go +++ b/pkg/sites/base.go @@ -9,7 +9,7 @@ type BaseSite interface { Initialize(comic *core.ComicIssue) error // GetInfo will return the comic name and issue number - GetInfo(url string) (string, string) + GetInfo(url string) (string, string, error) // RetrieveIssueLinks will return the images links of a comic RetrieveIssueLinks() ([]string, error) diff --git a/pkg/sites/comicextra.go b/pkg/sites/comicextra.go index 58d0636a..e76558b0 100644 --- a/pkg/sites/comicextra.go +++ b/pkg/sites/comicextra.go @@ -162,13 +162,13 @@ func (c *Comicextra) RetrieveIssueLinks() ([]string, error) { } // GetInfo extracts the basic info from the given url. -func (c *Comicextra) GetInfo(url string) (string, string) { +func (c *Comicextra) GetInfo(url string) (string, string, error) { parts := util.TrimAndSplitURL(url) name := parts[3] issueNumber := parts[4] - return name, issueNumber + return name, issueNumber, nil } // Initialize will initialize the comic based diff --git a/pkg/sites/loader.go b/pkg/sites/loader.go index 950d95d3..9b1a7f80 100644 --- a/pkg/sites/loader.go +++ b/pkg/sites/loader.go @@ -32,7 +32,12 @@ func initializeCollection(issues []string, options *config.Options, base BaseSit } for _, url := range issues { - name, issueNumber := base.GetInfo(url) + name, issueNumber, err := base.GetInfo(url) + if err != nil { + options.Logger.Errorf("error getting info for url %q: %v", url, err) + continue + } + name = util.Parse(name) if len(options.CustomComicName) > 0 { name = options.CustomComicName diff --git a/pkg/sites/loader_test.go b/pkg/sites/loader_test.go index dd2c1407..6f4d6fd1 100644 --- a/pkg/sites/loader_test.go +++ b/pkg/sites/loader_test.go @@ -23,11 +23,11 @@ func (s *stubSite) Initialize(comic *core.ComicIssue) error { return errors.New("missing comic") } -func (s *stubSite) GetInfo(url string) (string, string) { +func (s *stubSite) GetInfo(url string) (string, string, error) { if stub, ok := s.comics[url]; ok { - return stub.Name, stub.IssueNumber + return stub.Name, stub.IssueNumber, nil } - return "", "" + return "", "", errors.New("missing comic") } func (s *stubSite) RetrieveIssueLinks() ([]string, error) { diff --git a/pkg/sites/mangadex.go b/pkg/sites/mangadex.go index 146e1a9d..646462fc 100644 --- a/pkg/sites/mangadex.go +++ b/pkg/sites/mangadex.go @@ -409,16 +409,16 @@ func (m *Mangadex) RetrieveIssueLinks() ([]string, error) { } // GetInfo extracts the basic info from the given url. -func (m *Mangadex) GetInfo(urlValue string) (string, string) { +func (m *Mangadex) GetInfo(urlValue string) (string, string, error) { parts := util.TrimAndSplitURL(urlValue) if len(parts) < 5 { - return "", "" + return "", "", errors.New("URL not supported") } switch parts[3] { case "chapter": chapter, err := m.getChapterInfo(parts[4]) if err != nil { - return "", "" + return "", "", err } var chapterTitle string @@ -434,18 +434,18 @@ func (m *Mangadex) GetInfo(urlValue string) (string, string) { } manga, err := m.getMangaInfo(chapter.MangaID) if err != nil { - return "", chapterTitle + return "", "", err } - return manga.Title, chapterTitle + return manga.Title, chapterTitle, nil case "title": manga, err := m.getMangaInfo(parts[4]) if err != nil { - return "", "" + return "", "", err } - return manga.Title, "" + return manga.Title, "", nil default: - return "", "" + return "", "", errors.New("URL not supported") } } diff --git a/pkg/sites/mangadex_test.go b/pkg/sites/mangadex_test.go index 9221611e..5363b063 100644 --- a/pkg/sites/mangadex_test.go +++ b/pkg/sites/mangadex_test.go @@ -228,7 +228,8 @@ func TestMangadexGetInfoEnglish(t *testing.T) { md, cleanup := newTestMangadex(t, "series-1", "en") defer cleanup() - title, chapter := md.GetInfo(md.chapterBase + "/chapter-1") + title, chapter, err := md.GetInfo(md.chapterBase + "/chapter-1") + require.NoError(t, err) require.Equal(t, "Test Manga", title) require.Equal(t, "Vol 1 Chapter 1, Start", chapter) } @@ -237,7 +238,8 @@ func TestMangadexGetInfoJapanese(t *testing.T) { md, cleanup := newTestMangadex(t, "series-1", "jp") defer cleanup() - title, chapter := md.GetInfo(md.chapterBase + "/chapter-1") + title, chapter, err := md.GetInfo(md.chapterBase + "/chapter-1") + require.NoError(t, err) require.Equal(t, "テスト", title) require.Equal(t, "Vol 1 Chapter 1, Start", chapter) } @@ -246,7 +248,8 @@ func TestMangadexGetInfoNoCountry(t *testing.T) { md, cleanup := newTestMangadex(t, "series-1", "") defer cleanup() - title, chapter := md.GetInfo(md.chapterBase + "/chapter-1") + title, chapter, err := md.GetInfo(md.chapterBase + "/chapter-1") + require.NoError(t, err) require.Equal(t, "テスト", title) require.Equal(t, "Vol 1 Chapter 1, Start", chapter) } diff --git a/pkg/sites/mangakakalot.go b/pkg/sites/mangakakalot.go index 890b48fb..e4f381f5 100644 --- a/pkg/sites/mangakakalot.go +++ b/pkg/sites/mangakakalot.go @@ -17,14 +17,13 @@ func NewMangaKakalot(options *config.Options) *MangaKakalot { } // GetInfo extracts the basic info from the given url. -func (m *MangaKakalot) GetInfo(url string) (string, string) { +func (m *MangaKakalot) GetInfo(url string) (string, string, error) { name, issueNumber, err := MangaKakalotGetInfo(m.options, "mangakakalot.com", url) if err != nil { - m.options.Logger.Errorf("error getting info for url %q: %v", url, err) - return "", "" + return "", "", err } - return name, issueNumber + return name, issueNumber, nil } // Initialize loads links and metadata from mangakakalot diff --git a/pkg/sites/mangakakalot_test.go b/pkg/sites/mangakakalot_test.go index be5b4e74..dddb2565 100644 --- a/pkg/sites/mangakakalot_test.go +++ b/pkg/sites/mangakakalot_test.go @@ -76,7 +76,7 @@ func TestMangaKakalotScraper(t *testing.T) { } scraper := NewMangaKakalot(opts) - title, issue := scraper.GetInfo(server.URL + mangaKakalotChapterPath) + title, issue, err := scraper.GetInfo(server.URL + mangaKakalotChapterPath) require.Equal(t, "My Manga", title) require.Equal(t, "2", issue) diff --git a/pkg/sites/manganato.go b/pkg/sites/manganato.go index 7c53b9a9..f72bd61b 100644 --- a/pkg/sites/manganato.go +++ b/pkg/sites/manganato.go @@ -17,14 +17,14 @@ func NewManganato(options *config.Options) *Manganato { } // GetInfo extracts the basic info from the given url. -func (m *Manganato) GetInfo(url string) (string, string) { +func (m *Manganato) GetInfo(url string) (string, string, error) { name, issueNumber, err := MangaKakalotGetInfo(m.options, "manganato.com", url) if err != nil { - m.options.Logger.Errorf("error getting info for url %q: %v", url, err) - return "", "" + + return "", "", err } - return name, issueNumber + return name, issueNumber, nil } // Initialize loads links and metadata from manganato diff --git a/pkg/sites/manganato_test.go b/pkg/sites/manganato_test.go index 0e7f9ca9..03278dfe 100644 --- a/pkg/sites/manganato_test.go +++ b/pkg/sites/manganato_test.go @@ -74,7 +74,8 @@ func TestManganatoScraper(t *testing.T) { scraper := NewManganato(opts) - title, issue := scraper.GetInfo(server.URL + manganatoChapterPath) + title, issue, err := scraper.GetInfo(server.URL + manganatoChapterPath) + require.NoError(t, err) require.Equal(t, "My Manga", title) require.Equal(t, "2", issue) diff --git a/pkg/sites/mangareader.go b/pkg/sites/mangareader.go index 4e3767f9..838f1eb2 100644 --- a/pkg/sites/mangareader.go +++ b/pkg/sites/mangareader.go @@ -118,17 +118,21 @@ func (m *Mangareader) RetrieveIssueLinks() ([]string, error) { } // GetInfo extracts the basic info from the given URL. -func (m *Mangareader) GetInfo(url string) (string, string) { +func (m *Mangareader) GetInfo(url string) (string, string, error) { parts := util.TrimAndSplitURL(url) name := parts[3] issueNumber := parts[4] - return name, issueNumber + return name, issueNumber, nil } // Initialize loads links and metadata from mangareader func (m *Mangareader) Initialize(comic *core.ComicIssue) error { - name, issueNumber := m.GetInfo(comic.Source.URL) + name, issueNumber, err := m.GetInfo(comic.Source.URL) + if err != nil { + return err + } + comic.Name = name comic.IssueNumber = issueNumber diff --git a/pkg/sites/mangatown.go b/pkg/sites/mangatown.go index baf4c6bb..d27edf27 100644 --- a/pkg/sites/mangatown.go +++ b/pkg/sites/mangatown.go @@ -142,12 +142,12 @@ func (m *Mangatown) RetrieveIssueLinks() ([]string, error) { } // GetInfo extracts the basic info from the given URL. -func (m *Mangatown) GetInfo(url string) (string, string) { +func (m *Mangatown) GetInfo(url string) (string, string, error) { parts := util.TrimAndSplitURL(url) name := parts[4] issueNumber := parts[len(parts)-1] - return name, issueNumber + return name, issueNumber, nil } // Initialize loads links and metadata from mangatown diff --git a/pkg/sites/readallcomics.go b/pkg/sites/readallcomics.go index 649dca38..95a43998 100644 --- a/pkg/sites/readallcomics.go +++ b/pkg/sites/readallcomics.go @@ -189,14 +189,15 @@ func (r *Readallcomics) RetrieveIssueLinks() ([]string, error) { } // GetInfo extracts the comic info from the given URL. -func (r *Readallcomics) GetInfo(url string) (string, string) { +func (r *Readallcomics) GetInfo(url string) (string, string, error) { parts := util.TrimAndSplitURL(url) lastPart := parts[len(parts)-1] urlParts := strings.Split(lastPart, "-") // Handle simple case with no hyphens if len(urlParts) <= 1 { - return r.parseSimpleFormat(lastPart) + name, issueNumber := r.parseSimpleFormat(lastPart) + return name, issueNumber, nil } // Find potential issue number indices @@ -207,16 +208,19 @@ func (r *Readallcomics) GetInfo(url string) (string, string) { // Extract name and issue number based on split index if splitIndex > 0 { - return r.extractInfoWithSplitIndex(urlParts, splitIndex) + name, issueNumber := r.extractInfoWithSplitIndex(urlParts, splitIndex) + return name, issueNumber, nil } // Handle year suffix pattern (e.g., "name-issue-year") if r.hasYearSuffix(urlParts) { - return r.parseYearSuffixFormat(urlParts) + name, issueNumber := r.parseYearSuffixFormat(urlParts) + return name, issueNumber, nil } // Default fallback: last part is issue number - return r.parseDefaultFormat(urlParts) + name, issueNumber := r.parseDefaultFormat(urlParts) + return name, issueNumber, nil } // parseSimpleFormat handles URLs with no hyphens in the last part diff --git a/pkg/sites/readallcomics_test.go b/pkg/sites/readallcomics_test.go index 710b4704..e276c106 100644 --- a/pkg/sites/readallcomics_test.go +++ b/pkg/sites/readallcomics_test.go @@ -106,7 +106,8 @@ func TestReadAllComicsGetInfoParsing(t *testing.T) { } for _, tc := range tests { - name, issue := scraper.GetInfo(tc.url) + name, issue, err := scraper.GetInfo(tc.url) + require.NoError(t, err) require.Equal(t, tc.expectedName, name) require.Equal(t, tc.expectedIssue, issue) } diff --git a/pkg/sites/readcomiconline.go b/pkg/sites/readcomiconline.go index 8535ff81..a639bed8 100644 --- a/pkg/sites/readcomiconline.go +++ b/pkg/sites/readcomiconline.go @@ -200,12 +200,12 @@ func (c *ReadComicOnline) RetrieveIssueLinks() ([]string, error) { } // GetInfo extracts the basic info from the given url. -func (c *ReadComicOnline) GetInfo(url string) (string, string) { +func (c *ReadComicOnline) GetInfo(url string) (string, string, error) { parts := util.TrimAndSplitURL(url) name := parts[4] issueNumber := strings.Split(strings.ReplaceAll(parts[5], "Issue-", ""), "?")[0] - return name, issueNumber + return name, issueNumber, nil } // Initialize will initialize the comic based From a4427048cf42b9a1db84fe2e03687b3cfb6f4e12 Mon Sep 17 00:00:00 2001 From: ProjectDislocate <173194962+ProjectDislocate@users.noreply.github.com> Date: Sat, 7 Mar 2026 16:55:40 -0500 Subject: [PATCH 29/79] feat: add mangadex link to weblinks --- pkg/sites/mangadex.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pkg/sites/mangadex.go b/pkg/sites/mangadex.go index 646462fc..5cb5db76 100644 --- a/pkg/sites/mangadex.go +++ b/pkg/sites/mangadex.go @@ -18,7 +18,8 @@ import ( const ( mangadexAPIBase = "https://api.mangadex.org" - mangadexChapterBase = "https://mangadex.org/chapter" + mangadexWebBase = "https://mangadex.org" + mangadexChapterBase = mangadexWebBase + "/chapter" mangadexUploadsBase = "https://uploads.mangadex.org/data" ) @@ -187,6 +188,9 @@ func (m *Mangadex) getMangaInfo(mangaID string) (mangadexSeries, error) { manga.ContentRating = core.AgeRatingEveryone } + // set link to main mangadex page for the manga + manga.WebLinks = append(manga.WebLinks, fmt.Sprintf("%s/title/%s", mangadexWebBase, mangaID)) + // also add any additional links provided by mangadex for key, link := range mangaRes.Data.Attributes.Links { fullURL := m.mangaLinkToFullURL(key, link) if fullURL != "" { From 8d2dbefd58e9a866048ac9fde5a64fb79d87dfdb Mon Sep 17 00:00:00 2001 From: ProjectDislocate <173194962+ProjectDislocate@users.noreply.github.com> Date: Sat, 7 Mar 2026 21:17:47 -0500 Subject: [PATCH 30/79] feat: log msgs are no longer voided during debug by progress bar --- pkg/core/core.go | 15 ++++++++++++--- pkg/core/output.go | 10 +++++++--- pkg/sites/loader.go | 1 + 3 files changed, 20 insertions(+), 6 deletions(-) diff --git a/pkg/core/core.go b/pkg/core/core.go index 088a9108..4afea8fd 100644 --- a/pkg/core/core.go +++ b/pkg/core/core.go @@ -255,7 +255,11 @@ func (comic *ComicIssue) DownloadImages(options *config.Options) (*DownloadResul return nil, err } - progress := progressbar.NewOptions(len(comic.ImageLinks), progressbar.OptionSetRenderBlankState(true)) + var progress *progressbar.ProgressBar + if !options.Debug { + progress = progressbar.NewOptions(len(comic.ImageLinks), progressbar.OptionSetRenderBlankState(true)) + } + format := util.ImageType(comic.ImagesFormat) requestDelay := options.RequestDelay @@ -303,8 +307,13 @@ func (comic *ComicIssue) DownloadImages(options *config.Options) (*DownloadResul group.Go(func() error { defer sem.Release(1) defer func() { - if progressErr := progress.Add(1); progressErr != nil && options.Logger != nil { - options.Logger.Error(progressErr.Error()) + if progress != nil { + err := progress.Add(1) + if err != nil && options.Logger != nil { + options.Logger.Error(err.Error()) + } + } else if options.Logger != nil { + options.Logger.Infof("Downloaded image %d/%d", job.index+1, len(comic.ImageLinks)) } }() diff --git a/pkg/core/output.go b/pkg/core/output.go index 96c1ef85..e6d5c5db 100644 --- a/pkg/core/output.go +++ b/pkg/core/output.go @@ -55,7 +55,7 @@ func (comic *ComicIssue) makeComicInfoXML(options *config.Options, images *Downl } comicInfoPath := filepath.Join(outputDir, "ComicInfo.xml") - options.Logger.Infof("ComicInfo.xml path: %s", comicInfoPath) + options.Logger.Debugf("ComicInfo.xml path: %s", comicInfoPath) fo, err := os.Create(comicInfoPath) if err != nil { @@ -119,10 +119,14 @@ func (comic *ComicIssue) makeComicInfoXML(options *config.Options, images *Downl comicInfo.CreateElement("CommunityRating").SetText(fmt.Sprintf("%.2f", *comic.SeriesMetadata.CommunityRating)) } if len(comic.SeriesMetadata.Tags) > 0 { - comicInfo.CreateElement("Tags").SetText(strings.Join(comic.SeriesMetadata.Tags, ",")) + tags := strings.Join(comic.SeriesMetadata.Tags, ",") + options.Logger.Debugf("Adding tags to ComicInfo.xml: %s", tags) + comicInfo.CreateElement("Tags").SetText(tags) } if len(comic.SeriesMetadata.Genres) > 0 { - comicInfo.CreateElement("Genres").SetText(strings.Join(comic.SeriesMetadata.Genres, ",")) + genres := strings.Join(comic.SeriesMetadata.Genres, ",") + options.Logger.Debugf("Adding genres to ComicInfo.xml: %s", genres) + comicInfo.CreateElement("Genres").SetText(genres) } if len(comic.SeriesMetadata.WebLinks) > 0 { var cleanedWebLinks []string diff --git a/pkg/sites/loader.go b/pkg/sites/loader.go index 9b1a7f80..1bb9a694 100644 --- a/pkg/sites/loader.go +++ b/pkg/sites/loader.go @@ -76,6 +76,7 @@ func initializeCollection(issues []string, options *config.Options, base BaseSit Title: name, }, } + options.Logger.Debugf("Initializing comic with URL: %s", comic.Source.URL) if err = base.Initialize(comic); err != nil { return collection, err } From a1849ea9404df7f254cac93bf51bc2a09477471c Mon Sep 17 00:00:00 2001 From: ProjectDislocate <173194962+ProjectDislocate@users.noreply.github.com> Date: Sat, 7 Mar 2026 21:18:23 -0500 Subject: [PATCH 31/79] fix: duplicate AgeRating tag in comicinfo.xml --- pkg/core/output.go | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/pkg/core/output.go b/pkg/core/output.go index e6d5c5db..3c00524d 100644 --- a/pkg/core/output.go +++ b/pkg/core/output.go @@ -83,9 +83,14 @@ func (comic *ComicIssue) makeComicInfoXML(options *config.Options, images *Downl comicInfo.CreateElement("LocalizedSeries").SetText(localizedTitle) comicDescription := comic.getLocalizedDescription(options) - comicInfo.CreateElement("Summary").SetText(comicDescription) + if comicDescription != "" { + comicInfo.CreateElement("Summary").SetText(comicDescription) + } + + if comic.IssueNumber != "" { + comicInfo.CreateElement("Number").SetText(comic.IssueNumber) + } - comicInfo.CreateElement("Number").SetText(comic.IssueNumber) if comic.Volume != nil { comicInfo.CreateElement("Volume").SetText(*comic.Volume) } @@ -136,9 +141,6 @@ func (comic *ComicIssue) makeComicInfoXML(options *config.Options, images *Downl } comicInfo.CreateElement("WebLinks").SetText(strings.Join(cleanedWebLinks, " ")) } - if comic.SeriesMetadata.AgeRating != nil { - comicInfo.CreateElement("AgeRating").SetText(string(*comic.SeriesMetadata.AgeRating)) - } if len(comic.SeriesMetadata.Creators) > 0 { var writers []string var pencillers []string From 099b15beb51d2721765ffcca10ee5d0ba434bd57 Mon Sep 17 00:00:00 2001 From: ProjectDislocate <173194962+ProjectDislocate@users.noreply.github.com> Date: Sat, 7 Mar 2026 21:19:07 -0500 Subject: [PATCH 32/79] fix: loader tests failing --- pkg/sites/loader_test.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkg/sites/loader_test.go b/pkg/sites/loader_test.go index 6f4d6fd1..ee744bc9 100644 --- a/pkg/sites/loader_test.go +++ b/pkg/sites/loader_test.go @@ -4,6 +4,7 @@ import ( "errors" "testing" + "github.com/Girbons/comics-downloader/internal/logger" "github.com/Girbons/comics-downloader/pkg/config" "github.com/Girbons/comics-downloader/pkg/core" "github.com/stretchr/testify/assert" @@ -41,6 +42,7 @@ func TestInitializeCollectionFiltersIssues(t *testing.T) { ImagesFormat: "png", IssuesRange: "1-2", All: true, + Logger: logger.NewLogger(false, nil), } site := &stubSite{ @@ -60,7 +62,7 @@ func TestInitializeCollectionFiltersIssues(t *testing.T) { } func TestLoadComicFromSourceUnknown(t *testing.T) { - options := &config.Options{SourceName: "unknown"} + options := &config.Options{SourceName: "unknown", Logger: logger.NewLogger(false, nil)} collection, err := LoadComicFromSource(options) require.Error(t, err) require.Empty(t, collection) From f6165eb9b5ff692d2a9715dc2e7a5355ac840b83 Mon Sep 17 00:00:00 2001 From: ProjectDislocate <173194962+ProjectDislocate@users.noreply.github.com> Date: Sat, 7 Mar 2026 21:29:24 -0500 Subject: [PATCH 33/79] fix: mangadex not returning tags and genres --- pkg/sites/mangadex.go | 10 ++++++++-- pkg/sites/mangadex_test.go | 21 +++++++++++++++++++++ 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/pkg/sites/mangadex.go b/pkg/sites/mangadex.go index 5cb5db76..1e4f39bd 100644 --- a/pkg/sites/mangadex.go +++ b/pkg/sites/mangadex.go @@ -164,14 +164,20 @@ func (m *Mangadex) getMangaInfo(mangaID string) (mangadexSeries, error) { } } - if name != "" { + if name == "" { continue } + // classify tag groups switch tag.Attributes.Group { case "genre": manga.Genres = append(manga.Genres, name) - case "tag", "format": + case "tag", "content", "theme": + manga.Tags = append(manga.Tags, name) + case "format": + if name == "Oneshot" { + manga.IsOneShot = true + } manga.Tags = append(manga.Tags, name) } } diff --git a/pkg/sites/mangadex_test.go b/pkg/sites/mangadex_test.go index 5363b063..bb17d716 100644 --- a/pkg/sites/mangadex_test.go +++ b/pkg/sites/mangadex_test.go @@ -253,3 +253,24 @@ func TestMangadexGetInfoNoCountry(t *testing.T) { require.Equal(t, "テスト", title) require.Equal(t, "Vol 1 Chapter 1, Start", chapter) } + +func TestMangadexInitializeMetadataTagsGenres(t *testing.T) { + md, cleanup := newTestMangadex(t, "series-1", "en") + defer cleanup() + + comic := &core.ComicIssue{ + Source: &core.ComicSource{Name: "test-source", URL: md.chapterBase + "/chapter-1"}, + } + err := md.Initialize(comic) + require.NoError(t, err) + + // ensure metadata exists and contains expected genres and tags + require.NotNil(t, comic.SeriesMetadata) + require.Contains(t, comic.SeriesMetadata.Genres, "Romance") + // publicationDemographic should be added to Tags + require.Contains(t, comic.SeriesMetadata.Tags, "shounen") + // format tag should include Doujinshi + require.Contains(t, comic.SeriesMetadata.Tags, "Doujinshi") + // theme tags like "School Life" are not classified to Tags/Genres by current logic + require.Contains(t, comic.SeriesMetadata.Tags, "School Life") +} From 43338374f9678415361090394ea37452672432a0 Mon Sep 17 00:00:00 2001 From: ProjectDislocate <173194962+ProjectDislocate@users.noreply.github.com> Date: Sat, 7 Mar 2026 21:33:27 -0500 Subject: [PATCH 34/79] feat: set comic format in comicinfo.xml --- pkg/core/metadata.go | 24 ++++++++++++++++++++++++ pkg/core/output.go | 3 +++ pkg/sites/mangadex.go | 13 ++++++++++++- 3 files changed, 39 insertions(+), 1 deletion(-) diff --git a/pkg/core/metadata.go b/pkg/core/metadata.go index fed61870..fdd03b38 100644 --- a/pkg/core/metadata.go +++ b/pkg/core/metadata.go @@ -28,6 +28,29 @@ const ( AgeRatingX18 AgeRating = "X18+" ) +type ComicFormat string + +// Common comic formats +const ( + ComicFormatSpecial ComicFormat = "Special" + ComicFormatReference ComicFormat = "Reference" + ComicFormatDirectorsCut ComicFormat = "Director's Cut" + ComicFormatBoxSet ComicFormat = "Box Set" + ComicFormatAnnual ComicFormat = "Annual" + ComicFormatAnthology ComicFormat = "Anthology" + ComicFormatEpilogue ComicFormat = "Epilogue" + ComicFormatOneShot ComicFormat = "One-Shot" + ComicForamtPrologue ComicFormat = "Prologue" + ComicFormatTPB ComicFormat = "TPB" + ComicFormatTradePaperback ComicFormat = "Trade Paper Back" + ComicFormatOmnibus ComicFormat = "Omnibus" + ComicFormatCompendium ComicFormat = "Compendium" + ComicFormatAbsolute ComicFormat = "Absolute" + ComicFormatGraphicNovel ComicFormat = "Graphic Novel" + ComicFormatGN ComicFormat = "GN" + ComicFormatFCB ComicFormat = "FCB" +) + type ComicSource struct { Name string URL string // URL of the comic/manga issue @@ -65,6 +88,7 @@ type ComicIssue struct { Volume *string LanguageISO *string // IETF language tag ReleaseDate *time.Time + ComicFormat *ComicFormat ImageLinks []string OutputFormat ComicOutputFormat diff --git a/pkg/core/output.go b/pkg/core/output.go index 3c00524d..1decd4ec 100644 --- a/pkg/core/output.go +++ b/pkg/core/output.go @@ -197,6 +197,9 @@ func (comic *ComicIssue) makeComicInfoXML(options *config.Options, images *Downl } } comicInfo.CreateElement("PageCount").SetText(fmt.Sprintf("%d", len(images.FilePaths))) + if comic.ComicFormat != nil { + comicInfo.CreateElement("Format").SetText(string(*comic.ComicFormat)) + } doc.Indent(2) _, err = doc.WriteTo(fo) diff --git a/pkg/sites/mangadex.go b/pkg/sites/mangadex.go index 1e4f39bd..89d9a1e1 100644 --- a/pkg/sites/mangadex.go +++ b/pkg/sites/mangadex.go @@ -68,6 +68,8 @@ type mangadexSeries struct { Tags []string Genres []string WebLinks []string + + IsOneShot bool } func (m *Mangadex) getMangaInfo(mangaID string) (mangadexSeries, error) { @@ -99,6 +101,10 @@ func (m *Mangadex) getMangaInfo(mangaID string) (mangadexSeries, error) { } `json:"tags"` // AvailableTranslatedLanguages []string `json:"availableTranslatedLanguages"` } `json:"attributes"` + Relationships []struct { + ID string `json:"id"` + Type string `json:"type"` // e.g. "author", "artist", "cover_art" + } `json:"relationships"` } `json:"data"` } @@ -380,7 +386,6 @@ func (m *Mangadex) getChapterInfo(chapterID string) (chapterInfo mangadexChapter var chapterNumber string var chapterTitle string if chapterRes.Data.Attributes.Chapter != nil { - // TODO: consider defaulting to "oneshot"? chapterNumber = *chapterRes.Data.Attributes.Chapter } if chapterRes.Data.Attributes.Title != nil { @@ -486,6 +491,7 @@ func (m *Mangadex) Initialize(comic *core.ComicIssue) error { comic.LanguageISO = &chapter.TranslatedLanguage comic.ReleaseDate = &chapter.PublishAt + // ensure metadata object exists if comic.SeriesMetadata == nil { comic.SeriesMetadata = &core.SeriesMetadata{} } @@ -498,6 +504,11 @@ func (m *Mangadex) Initialize(comic *core.ComicIssue) error { comic.SeriesMetadata.WebLinks = manga.WebLinks comic.SeriesMetadata.IsManga = &manga.IsManga + if manga.IsOneShot { + format := core.ComicFormatOneShot + comic.ComicFormat = &format + } + comic.ImageLinks = chapter.ImageLinks return nil From 5d03d9f1d6b580f3be614b2320e29f4797b5880b Mon Sep 17 00:00:00 2001 From: ProjectDislocate <173194962+ProjectDislocate@users.noreply.github.com> Date: Sat, 7 Mar 2026 22:05:52 -0500 Subject: [PATCH 35/79] feat: set author info from mangadex --- pkg/core/metadata.go | 1 + pkg/sites/mangadex.go | 140 ++++++++++++++++++++++++++++++++----- pkg/sites/mangadex_test.go | 44 ++++++++++-- 3 files changed, 162 insertions(+), 23 deletions(-) diff --git a/pkg/core/metadata.go b/pkg/core/metadata.go index fdd03b38..7885b4d2 100644 --- a/pkg/core/metadata.go +++ b/pkg/core/metadata.go @@ -68,6 +68,7 @@ type SeriesMetadata struct { LocalizedTitle map[string]string // Map of language code to localized title, e.g. {"en": "One Piece", "jp": "ワンピース"} Description map[string]string // Map of language code to description, e.g. {"en": "A story about pirates...", "jp": "海賊の物語..."} Creators []SeriesCreator + CoverURL *string IsManga *bool // True if it's a manga, false if it's a comic IsRTL *bool // True if the comic/manga is read right-to-left, false if left-to-right. Only relevant for manga, but some comics may also be RTL. diff --git a/pkg/sites/mangadex.go b/pkg/sites/mangadex.go index 89d9a1e1..cc5faf07 100644 --- a/pkg/sites/mangadex.go +++ b/pkg/sites/mangadex.go @@ -17,31 +17,35 @@ import ( ) const ( - mangadexAPIBase = "https://api.mangadex.org" - mangadexWebBase = "https://mangadex.org" - mangadexChapterBase = mangadexWebBase + "/chapter" - mangadexUploadsBase = "https://uploads.mangadex.org/data" + mangadexWebBase = "https://mangadex.org" + mangadexAPIBase = "https://api.mangadex.org" + mangadexUploadsBase = "https://uploads.mangadex.org" + mangadexChapterBase = mangadexWebBase + "/chapter" + mangadexUploadsData = mangadexUploadsBase + "/data" + mangadexUploadsCovers = mangadexUploadsBase + "/covers" ) // Mangadex represents a mangadex instance. type Mangadex struct { - country string - options *config.Options - client *httpclient.ComicClient - apiBase string - chapterBase string - uploadsBase string + country string + options *config.Options + client *httpclient.ComicClient + apiBase string + chapterBase string + uploadsData string + uploadsCovers string } // NewMangadex returns a Mangadex instance. func NewMangadex(options *config.Options) *Mangadex { return &Mangadex{ - country: strings.ToLower(options.Country), - options: options, - client: options.Client, - apiBase: mangadexAPIBase, - chapterBase: mangadexChapterBase, - uploadsBase: mangadexUploadsBase, + country: strings.ToLower(options.Country), + options: options, + client: options.Client, + apiBase: mangadexAPIBase, + chapterBase: mangadexChapterBase, + uploadsData: mangadexUploadsData, + uploadsCovers: mangadexUploadsCovers, } } @@ -59,19 +63,71 @@ type mangadexSeries struct { Title string // the title we want to use for the series, which may be localized based on the country option LocalizedTitle map[string]string Description map[string]string - IsManga bool + IsManga bool OriginalLanguage string Year *int + CoverURL *string ContentRating core.AgeRating Tags []string Genres []string WebLinks []string + Authors []string + Artists []string + IsOneShot bool } +func (m *Mangadex) getMangaCoverURL(mangaID, coverID string) (string, error) { + ctx, cancel := m.requestContext() + defer cancel() + + endpoint := joinURL(m.apiBase, fmt.Sprintf("/cover/%s", coverID)) + var coverRes struct { + Result string `json:"result"` + Data struct { + Attributes struct { + FileName string `json:"fileName"` + } `json:"attributes"` + } `json:"data"` + } + + if err := fetchJSON(ctx, m.client, endpoint, &coverRes); err != nil { + return "", err + } + if strings.ToLower(coverRes.Result) != "ok" { + return "", fmt.Errorf("unexpected response") + } + + return joinURL(m.uploadsCovers, fmt.Sprintf("%s/%s", mangaID, coverRes.Data.Attributes.FileName)), nil +} + +func (m *Mangadex) getAuthorInfo(authorID string) (string, error) { + ctx, cancel := m.requestContext() + defer cancel() + + endpoint := joinURL(m.apiBase, fmt.Sprintf("/author/%s", authorID)) + var authorRes struct { + Result string `json:"result"` + Data struct { + Attributes struct { + Name string `json:"name"` + } `json:"attributes"` + } `json:"data"` + } + + if err := fetchJSON(ctx, m.client, endpoint, &authorRes); err != nil { + return "", err + } + if strings.ToLower(authorRes.Result) != "ok" { + return "", fmt.Errorf("unexpected response") + } + + return authorRes.Data.Attributes.Name, nil +} + func (m *Mangadex) getMangaInfo(mangaID string) (mangadexSeries, error) { ctx, cancel := m.requestContext() defer cancel() @@ -121,6 +177,41 @@ func (m *Mangadex) getMangaInfo(mangaID string) (mangadexSeries, error) { IsManga: true, // default to true since it's a manga site } + // get author, artist, and cover info from relationships + cachedAuthors := map[string]string{} // cache author info to avoid duplicate requests + for _, rel := range mangaRes.Data.Relationships { + switch rel.Type { + case "cover_art": + coverURL, err := m.getMangaCoverURL(mangaID, rel.ID) + if err != nil { + return mangadexSeries{}, err + } + manga.CoverURL = &coverURL + case "author": + var err error + authorName, ok := cachedAuthors[rel.ID] + if !ok { + authorName, err = m.getAuthorInfo(rel.ID) + if err != nil { + return mangadexSeries{}, err + } + } + manga.Authors = append(manga.Authors, authorName) + cachedAuthors[rel.ID] = authorName + case "artist": + var err error + artistName, ok := cachedAuthors[rel.ID] + if !ok { + artistName, err = m.getAuthorInfo(rel.ID) + if err != nil { + return mangadexSeries{}, err + } + } + manga.Artists = append(manga.Artists, artistName) + cachedAuthors[rel.ID] = artistName + } + } + // Set titles foundTitle := false for lang, title := range mangaRes.Data.Attributes.Title { @@ -366,7 +457,7 @@ func (m *Mangadex) getChapterInfo(chapterID string) (chapterInfo mangadexChapter var imageLinks []string for _, file := range imagesRes.Chapter.Data { - imageURL := joinURL(m.uploadsBase, fmt.Sprintf("%s/%s", imagesRes.Chapter.Hash, file)) + imageURL := joinURL(m.uploadsData, fmt.Sprintf("%s/%s", imagesRes.Chapter.Hash, file)) imageLinks = append(imageLinks, imageURL) } @@ -509,6 +600,19 @@ func (m *Mangadex) Initialize(comic *core.ComicIssue) error { comic.ComicFormat = &format } + for _, author := range manga.Authors { + comic.SeriesMetadata.Creators = append(comic.SeriesMetadata.Creators, core.SeriesCreator{ + Name: author, + Role: core.CreatorRoleWriter, + }) + } + for _, artist := range manga.Artists { + comic.SeriesMetadata.Creators = append(comic.SeriesMetadata.Creators, core.SeriesCreator{ + Name: artist, + Role: core.CreatorRolePenciller, + }) + } + comic.ImageLinks = chapter.ImageLinks return nil diff --git a/pkg/sites/mangadex_test.go b/pkg/sites/mangadex_test.go index bb17d716..51c955d6 100644 --- a/pkg/sites/mangadex_test.go +++ b/pkg/sites/mangadex_test.go @@ -174,7 +174,7 @@ func newTestMangadex(t *testing.T, series, lang string) (*Mangadex, func()) { md := NewMangadex(opts) md.apiBase = server.URL md.chapterBase = server.URL + "/chapter" - md.uploadsBase = server.URL + "/data" + md.uploadsData = server.URL + "/data" cleanup := func() { server.Close() @@ -204,8 +204,8 @@ func TestMangadexInitialize(t *testing.T) { err := md.Initialize(comic) require.NoError(t, err) require.Equal(t, []string{ - md.uploadsBase + "/HASH/001.png", - md.uploadsBase + "/HASH/002.png", + md.uploadsData + "/HASH/001.png", + md.uploadsData + "/HASH/002.png", }, comic.ImageLinks) } @@ -219,8 +219,8 @@ func TestMangadexInitializeMinimum(t *testing.T) { err := md.Initialize(comic) require.NoError(t, err) require.Equal(t, []string{ - md.uploadsBase + "/HASH/001.png", - md.uploadsBase + "/HASH/002.png", + md.uploadsData + "/HASH/001.png", + md.uploadsData + "/HASH/002.png", }, comic.ImageLinks) } @@ -274,3 +274,37 @@ func TestMangadexInitializeMetadataTagsGenres(t *testing.T) { // theme tags like "School Life" are not classified to Tags/Genres by current logic require.Contains(t, comic.SeriesMetadata.Tags, "School Life") } + +func TestMangadexGetAuthorInfo(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case strings.HasPrefix(r.URL.Path, "/author/author-1"): + w.Header().Set("Content-Type", "application/json") + fmt.Fprint(w, `{"result":"ok","data":{"attributes":{"name":"John Doe"}}}`) + default: + http.NotFound(w, r) + } + })) + defer server.Close() + + client := httpclient.NewComicClient( + httpclient.WithHTTPClient(server.Client()), + httpclient.WithRetry(0, 0), + ) + + opts := &config.Options{ + URL: server.URL, + Country: "", + SourceName: "mangadex.org", + Logger: logger.NewLogger(false, nil), + Client: client, + RequestTimeout: config.DefaulltRequestTimeout, + } + + md := NewMangadex(opts) + md.apiBase = server.URL + + name, err := md.getAuthorInfo("author-1") + require.NoError(t, err) + require.Equal(t, "John Doe", name) +} From 369c056ec3b87bb261afb6711fc2091f6adea52b Mon Sep 17 00:00:00 2001 From: ProjectDislocate <173194962+ProjectDislocate@users.noreply.github.com> Date: Sat, 7 Mar 2026 22:26:39 -0500 Subject: [PATCH 36/79] feat: mangadex skip unavailable chapters --- pkg/sites/mangadex.go | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/pkg/sites/mangadex.go b/pkg/sites/mangadex.go index cc5faf07..1c6d879a 100644 --- a/pkg/sites/mangadex.go +++ b/pkg/sites/mangadex.go @@ -365,9 +365,12 @@ func (m *Mangadex) getChapters(mangaID string) ([]string, error) { var chaptersRes struct { Result string `json:"result"` Volumes map[string]struct { + Volume string `json:"volume"` // volume name, or "none" + Count int `json:"count"` Chapters map[string]struct { - ID string `json:"id"` - Name string `json:"chapter"` + Chapter string `json:"chapter"` // the chapter number, not the chapter name + ID string `json:"id"` + IsUnavailable bool `json:"isUnavailable"` } `json:"chapters"` } `json:"volumes"` } @@ -382,6 +385,10 @@ func (m *Mangadex) getChapters(mangaID string) ([]string, error) { var ids []string for _, v := range chaptersRes.Volumes { for _, c := range v.Chapters { + if c.IsUnavailable { + continue + } + ids = append(ids, joinURL(m.chapterBase, c.ID)) } } @@ -613,6 +620,7 @@ func (m *Mangadex) Initialize(comic *core.ComicIssue) error { }) } + comic.SeriesMetadata.CoverURL = manga.CoverURL comic.ImageLinks = chapter.ImageLinks return nil From 3b7f079dbe56a6678a3005cc9a25404bf37fc085 Mon Sep 17 00:00:00 2001 From: ProjectDislocate <173194962+ProjectDislocate@users.noreply.github.com> Date: Sat, 7 Mar 2026 23:13:26 -0500 Subject: [PATCH 37/79] feat: ratings & localized titles from mangadex --- pkg/sites/mangadex.go | 116 ++++++++++++++++++++++++++----------- pkg/sites/mangadex_test.go | 30 ++++++++++ 2 files changed, 111 insertions(+), 35 deletions(-) diff --git a/pkg/sites/mangadex.go b/pkg/sites/mangadex.go index 1c6d879a..34c668cb 100644 --- a/pkg/sites/mangadex.go +++ b/pkg/sites/mangadex.go @@ -68,6 +68,7 @@ type mangadexSeries struct { OriginalLanguage string Year *int CoverURL *string + Rating *float64 ContentRating core.AgeRating Tags []string @@ -128,6 +129,36 @@ func (m *Mangadex) getAuthorInfo(authorID string) (string, error) { return authorRes.Data.Attributes.Name, nil } +func (m *Mangadex) getMangaRating(mangaID string) (float64, error) { + ctx, cancel := m.requestContext() + defer cancel() + + // need to do weird [] syntax so it counts as an array to the mangadex API + endpoint := joinURL(m.apiBase, fmt.Sprintf("/statistics/manga?manga[]=%s", mangaID)) + var ratingRes struct { + Result string `json:"result"` + Statistics map[string]struct { + Rating struct { + Bayesian float64 `json:"bayesian"` + } `json:"rating"` + } `json:"statistics"` + } + if err := fetchJSON(ctx, m.client, endpoint, &ratingRes); err != nil { + return 0, err + } + if strings.ToLower(ratingRes.Result) != "ok" { + return 0, fmt.Errorf("unexpected response") + } + + stats, exists := ratingRes.Statistics[mangaID] + if !exists { + return 0, fmt.Errorf("no statistics found for manga ID %s", mangaID) + } + + return stats.Rating.Bayesian, nil + +} + func (m *Mangadex) getMangaInfo(mangaID string) (mangadexSeries, error) { ctx, cancel := m.requestContext() defer cancel() @@ -155,7 +186,7 @@ func (m *Mangadex) getMangaInfo(mangaID string) (mangadexSeries, error) { Version int `json:"version"` } `json:"attributes"` } `json:"tags"` - // AvailableTranslatedLanguages []string `json:"availableTranslatedLanguages"` + AvailableTranslatedLanguages []string `json:"availableTranslatedLanguages"` } `json:"attributes"` Relationships []struct { ID string `json:"id"` @@ -177,6 +208,46 @@ func (m *Mangadex) getMangaInfo(mangaID string) (mangadexSeries, error) { IsManga: true, // default to true since it's a manga site } + // Set titles + foundTitle := false + for lang, title := range mangaRes.Data.Attributes.Title { + if m.country == "" || m.country == strings.ToLower(lang) { + manga.Title = title + foundTitle = true + break + } + + // manga.LocalizedTitle[lang] = title + } + + // try and fill in any missing localized titles + for _, grouping := range mangaRes.Data.Attributes.AltTitles { + for lang, title := range grouping { + if _, exists := manga.LocalizedTitle[lang]; !exists { + manga.LocalizedTitle[lang] = title + } + + // if main title is missing, try to use the alt titles to fill it + if !foundTitle && (m.country == "" || m.country == strings.ToLower(lang)) { + manga.Title = title + foundTitle = true + } + } + } + + if !foundTitle { + // If still haven't found anything fallback to any available title. + for _, title := range mangaRes.Data.Attributes.Title { + manga.Title = title + } + } + + // TODO: should we error if the manga isn't available in the specified country? + // previously we have just let the process continue + // if m.country != "" && !slices.Contains(mangaRes.Data.Attributes.AvailableTranslatedLanguages, strings.ToLower(m.country)) { + // return mangadexSeries{}, fmt.Errorf("manga \"%s\" not available in country %s", manga.Title, m.country) + // } + // get author, artist, and cover info from relationships cachedAuthors := map[string]string{} // cache author info to avoid duplicate requests for _, rel := range mangaRes.Data.Relationships { @@ -212,39 +283,12 @@ func (m *Mangadex) getMangaInfo(mangaID string) (mangadexSeries, error) { } } - // Set titles - foundTitle := false - for lang, title := range mangaRes.Data.Attributes.Title { - if m.country == "" || m.country == strings.ToLower(lang) { - manga.Title = title - foundTitle = true - break - } - - // manga.LocalizedTitle[lang] = title - } - - // try and fill in any missing localized titles - for _, grouping := range mangaRes.Data.Attributes.AltTitles { - for lang, title := range grouping { - if _, exists := manga.LocalizedTitle[lang]; !exists { - manga.LocalizedTitle[lang] = title - } - - // if main title is missing, try to use the alt titles to fill it - if !foundTitle && (m.country == "" || m.country == strings.ToLower(lang)) { - manga.Title = title - foundTitle = true - } - } - } - - if !foundTitle { - // If still haven't found anything fallback to any available title. - for _, title := range mangaRes.Data.Attributes.Title { - manga.Title = title - } + // get rating info + rating, err := m.getMangaRating(mangaID) + if err != nil { + return mangadexSeries{}, err } + manga.Rating = &rating // handle tags and genres if mangaRes.Data.Attributes.PublicationDemographic != nil { @@ -594,13 +638,15 @@ func (m *Mangadex) Initialize(comic *core.ComicIssue) error { comic.SeriesMetadata = &core.SeriesMetadata{} } - comic.SeriesMetadata.AgeRating = &manga.ContentRating + comic.SeriesMetadata.Title = manga.Title + comic.SeriesMetadata.LocalizedTitle = manga.LocalizedTitle comic.SeriesMetadata.Description = manga.Description comic.SeriesMetadata.Genres = manga.Genres comic.SeriesMetadata.Tags = manga.Tags - comic.SeriesMetadata.Title = manga.Title + comic.SeriesMetadata.AgeRating = &manga.ContentRating comic.SeriesMetadata.WebLinks = manga.WebLinks comic.SeriesMetadata.IsManga = &manga.IsManga + comic.SeriesMetadata.CommunityRating = manga.Rating if manga.IsOneShot { format := core.ComicFormatOneShot diff --git a/pkg/sites/mangadex_test.go b/pkg/sites/mangadex_test.go index 51c955d6..7811d0ca 100644 --- a/pkg/sites/mangadex_test.go +++ b/pkg/sites/mangadex_test.go @@ -15,8 +15,20 @@ import ( ) func setupMangadexServer() *httptest.Server { + hasMangaParam := func(r *http.Request, series string) bool { + for _, id := range r.URL.Query()["manga[]"] { + if id == series { + return true + } + } + return false + } + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch { + case r.URL.Path == "/statistics/manga" && hasMangaParam(r, "series-1"): + w.Header().Set("Content-Type", "application/json") + fmt.Fprint(w, `{"result":"ok","statistics":{"series-1":{"rating":{"bayesian":4.5}}}}`) case strings.HasPrefix(r.URL.Path, "/manga/series-1/aggregate"): w.Header().Set("Content-Type", "application/json") fmt.Fprint(w, `{ @@ -100,6 +112,9 @@ func setupMangadexServer() *httptest.Server { "result":"ok", "chapter":{"hash":"HASH","data":["001.png","002.png"]} }`) + case r.URL.Path == "/statistics/manga" && hasMangaParam(r, "series-2"): + w.Header().Set("Content-Type", "application/json") + fmt.Fprint(w, `{"result":"ok","statistics":{"series-1":{"rating":{"bayesian":4.5}}}}`) case strings.HasPrefix(r.URL.Path, "/manga/series-2/aggregate"): w.Header().Set("Content-Type", "application/json") fmt.Fprint(w, `{ @@ -275,6 +290,21 @@ func TestMangadexInitializeMetadataTagsGenres(t *testing.T) { require.Contains(t, comic.SeriesMetadata.Tags, "School Life") } +func TestMangadexInitializeMetadataRating(t *testing.T) { + md, cleanup := newTestMangadex(t, "series-1", "en") + defer cleanup() + + comic := &core.ComicIssue{ + Source: &core.ComicSource{Name: "test-source", URL: md.chapterBase + "/chapter-1"}, + } + err := md.Initialize(comic) + require.NoError(t, err) + + require.NotNil(t, comic.SeriesMetadata) + require.NotNil(t, comic.SeriesMetadata.CommunityRating) + require.Equal(t, 4.5, *comic.SeriesMetadata.CommunityRating) +} + func TestMangadexGetAuthorInfo(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch { From 2126df47f194e5e2dacc482b95f6c14061a04536 Mon Sep 17 00:00:00 2001 From: ProjectDislocate <173194962+ProjectDislocate@users.noreply.github.com> Date: Sun, 8 Mar 2026 21:42:29 -0400 Subject: [PATCH 38/79] fix: using wrong tag for links --- pkg/core/output.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/core/output.go b/pkg/core/output.go index 1decd4ec..398a9ca6 100644 --- a/pkg/core/output.go +++ b/pkg/core/output.go @@ -139,7 +139,7 @@ func (comic *ComicIssue) makeComicInfoXML(options *config.Options, images *Downl // the links must be URL-encoded as spaces are the separator cleanedWebLinks = append(cleanedWebLinks, url.QueryEscape(link)) } - comicInfo.CreateElement("WebLinks").SetText(strings.Join(cleanedWebLinks, " ")) + comicInfo.CreateElement("Web").SetText(strings.Join(cleanedWebLinks, " ")) } if len(comic.SeriesMetadata.Creators) > 0 { var writers []string From 6625cc9903f7e24a60664e3a5d12329189ea0f0d Mon Sep 17 00:00:00 2001 From: ProjectDislocate <173194962+ProjectDislocate@users.noreply.github.com> Date: Sun, 8 Mar 2026 21:48:52 -0400 Subject: [PATCH 39/79] fix: invalid urls being saved --- pkg/core/output.go | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/pkg/core/output.go b/pkg/core/output.go index 398a9ca6..ebefd2e8 100644 --- a/pkg/core/output.go +++ b/pkg/core/output.go @@ -3,7 +3,6 @@ package core import ( "errors" "fmt" - "net/url" "os" "path/filepath" "strings" @@ -134,12 +133,7 @@ func (comic *ComicIssue) makeComicInfoXML(options *config.Options, images *Downl comicInfo.CreateElement("Genres").SetText(genres) } if len(comic.SeriesMetadata.WebLinks) > 0 { - var cleanedWebLinks []string - for _, link := range comic.SeriesMetadata.WebLinks { - // the links must be URL-encoded as spaces are the separator - cleanedWebLinks = append(cleanedWebLinks, url.QueryEscape(link)) - } - comicInfo.CreateElement("Web").SetText(strings.Join(cleanedWebLinks, " ")) + comicInfo.CreateElement("Web").SetText(strings.Join(comic.SeriesMetadata.WebLinks, " ")) } if len(comic.SeriesMetadata.Creators) > 0 { var writers []string From a4598ce28ae20bff2fa3d19ff2befa55d55dd87f Mon Sep 17 00:00:00 2001 From: ProjectDislocate <173194962+ProjectDislocate@users.noreply.github.com> Date: Mon, 9 Mar 2026 10:21:35 -0400 Subject: [PATCH 40/79] fix: genre output tag --- pkg/core/output.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/core/output.go b/pkg/core/output.go index ebefd2e8..1f495fb6 100644 --- a/pkg/core/output.go +++ b/pkg/core/output.go @@ -130,7 +130,7 @@ func (comic *ComicIssue) makeComicInfoXML(options *config.Options, images *Downl if len(comic.SeriesMetadata.Genres) > 0 { genres := strings.Join(comic.SeriesMetadata.Genres, ",") options.Logger.Debugf("Adding genres to ComicInfo.xml: %s", genres) - comicInfo.CreateElement("Genres").SetText(genres) + comicInfo.CreateElement("Genre").SetText(genres) } if len(comic.SeriesMetadata.WebLinks) > 0 { comicInfo.CreateElement("Web").SetText(strings.Join(comic.SeriesMetadata.WebLinks, " ")) From 0b41e10b6c9d1758145b5c5b4fb62ed6d9a81099 Mon Sep 17 00:00:00 2001 From: ProjectDislocate <173194962+ProjectDislocate@users.noreply.github.com> Date: Mon, 9 Mar 2026 19:35:09 -0400 Subject: [PATCH 41/79] feat: show issue num on progress bar --- cmd/app/downloader.go | 2 +- pkg/core/core.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/app/downloader.go b/cmd/app/downloader.go index 910cf2bb..49ac5179 100644 --- a/cmd/app/downloader.go +++ b/cmd/app/downloader.go @@ -163,7 +163,7 @@ func (r *Runner) download(base config.Options) { continue } - perURL.Logger.Info("Downloading...") + perURL.Logger.Info("Downloading... " + trimmedURL) collection, err := sites.LoadComicFromSource(&perURL) if err != nil { perURL.Logger.Error(err.Error()) diff --git a/pkg/core/core.go b/pkg/core/core.go index 4afea8fd..f7f8ee81 100644 --- a/pkg/core/core.go +++ b/pkg/core/core.go @@ -257,7 +257,7 @@ func (comic *ComicIssue) DownloadImages(options *config.Options) (*DownloadResul var progress *progressbar.ProgressBar if !options.Debug { - progress = progressbar.NewOptions(len(comic.ImageLinks), progressbar.OptionSetRenderBlankState(true)) + progress = progressbar.NewOptions(len(comic.ImageLinks), progressbar.OptionSetRenderBlankState(true), progressbar.OptionSetDescription(fmt.Sprintf("#%s", comic.IssueNumber))) } format := util.ImageType(comic.ImagesFormat) From 2edc342de0c0e9b59ab9b57a195220e2147af103 Mon Sep 17 00:00:00 2001 From: ProjectDislocate <173194962+ProjectDislocate@users.noreply.github.com> Date: Mon, 9 Mar 2026 19:36:18 -0400 Subject: [PATCH 42/79] chore: save todo items --- pkg/sites/mangadex.go | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/pkg/sites/mangadex.go b/pkg/sites/mangadex.go index 34c668cb..dcfb3249 100644 --- a/pkg/sites/mangadex.go +++ b/pkg/sites/mangadex.go @@ -412,9 +412,11 @@ func (m *Mangadex) getChapters(mangaID string) ([]string, error) { Volume string `json:"volume"` // volume name, or "none" Count int `json:"count"` Chapters map[string]struct { - Chapter string `json:"chapter"` // the chapter number, not the chapter name - ID string `json:"id"` - IsUnavailable bool `json:"isUnavailable"` + Chapter string `json:"chapter"` // the chapter number, not the chapter name + ID string `json:"id"` + IsUnavailable bool `json:"isUnavailable"` + Others []string `json:"others"` // list of alternative chapter IDs for the same chapter (e.g. for different languages) + // TODO: add check to try and find target language in others if the main chapter is not what we want } `json:"chapters"` } `json:"volumes"` } @@ -458,6 +460,10 @@ func (m *Mangadex) getChapterInfo(chapterID string) (chapterInfo mangadexChapter ctx, cancel := m.requestContext() defer cancel() + // TODO: set total number of chapters available in the manga + // ~~TODO: set volume number in metadata if available~~ Seems to be a non-issue?? + // ~~TODO: set issue number to chapterNum~~ again seesm to be a non-issue? + endpoint := joinURL(m.apiBase, fmt.Sprintf("/chapter/%s", chapterID)) var chapterRes struct { Result string `json:"result"` @@ -608,12 +614,17 @@ func (m *Mangadex) GetInfo(urlValue string) (string, string, error) { // Initialize loads links and metadata from mangadex. func (m *Mangadex) Initialize(comic *core.ComicIssue) error { + + // TODO: fix localized title being set as "Series Title" in metadata + // TODO: fix chapter title being set to a fs compatible sanatized version of the chapter name instead of the actual chapter name + if comic == nil { return fmt.Errorf("comic is nil") } if comic.Source == nil { return fmt.Errorf("comic source is nil") } + parts := util.TrimAndSplitURL(comic.Source.URL) if len(parts) < 5 { return fmt.Errorf("URL not supported") From b9bcff9b42262a0a7c52a5cac946a47fe301e933 Mon Sep 17 00:00:00 2001 From: ProjectDislocate <173194962+ProjectDislocate@users.noreply.github.com> Date: Tue, 21 Apr 2026 16:42:52 -0400 Subject: [PATCH 43/79] chore: ignore patch files --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 99afb183..a2752cac 100644 --- a/.gitignore +++ b/.gitignore @@ -11,7 +11,7 @@ # Output of the go coverage tool, specifically when used with LiteIDE *.out - +*.patch build/ # path where comics are downloaded From eab117e4c95675c74e051adf91dccae72f781a2c Mon Sep 17 00:00:00 2001 From: ProjectDislocate <173194962+ProjectDislocate@users.noreply.github.com> Date: Tue, 21 Apr 2026 16:44:16 -0400 Subject: [PATCH 44/79] feat: set custom header per request --- cmd/app/downloader.go | 5 ++-- pkg/detector/detector.go | 4 +-- pkg/detector/detector_test.go | 2 +- pkg/http/client.go | 4 +++ pkg/sites/http_helpers.go | 49 ++++++++++++++++++++++++++++++----- 5 files changed, 51 insertions(+), 13 deletions(-) diff --git a/cmd/app/downloader.go b/cmd/app/downloader.go index 49ac5179..63589366 100644 --- a/cmd/app/downloader.go +++ b/cmd/app/downloader.go @@ -147,13 +147,12 @@ func (r *Runner) download(base config.Options) { perURL := opts perURL.URL = trimmedURL perURL.OutputFolder = outputFolder - // check if the link is supported - source, check, isDisabled := detector.DetectComic(trimmedURL) + source, isSupported, isDisabled := detector.DetectSource(trimmedURL) perURL.SourceName = source - if !check { + if !isSupported { perURL.Logger.Error("This site is not supported") continue } diff --git a/pkg/detector/detector.go b/pkg/detector/detector.go index 5a966dce..311c846c 100644 --- a/pkg/detector/detector.go +++ b/pkg/detector/detector.go @@ -19,8 +19,8 @@ var SupportedSites = map[string]map[string]bool{ "readcomiconline": {"isDisabled": false}, } -// DetectComic will look for the url source to check if a source is supported. -func DetectComic(url string) (string, bool, bool) { +// DetectSource will look for the url source to check if a source is supported. +func DetectSource(url string) (string, bool, bool) { var ( isSupported bool isDisabled bool diff --git a/pkg/detector/detector_test.go b/pkg/detector/detector_test.go index bcb85b07..33ce9983 100644 --- a/pkg/detector/detector_test.go +++ b/pkg/detector/detector_test.go @@ -7,7 +7,7 @@ import ( ) func TestUnsupportedSource(t *testing.T) { - _, check, isDisabled := DetectComic("http://example.com") + _, check, isDisabled := DetectSource("http://example.com") assert.False(t, check) assert.False(t, isDisabled) diff --git a/pkg/http/client.go b/pkg/http/client.go index 87c47d28..91767698 100644 --- a/pkg/http/client.go +++ b/pkg/http/client.go @@ -81,6 +81,7 @@ func WithHeaders(headers map[string]string) Option { if strings.TrimSpace(key) == "" || strings.TrimSpace(value) == "" { continue } + key = strings.ToLower(key) cc.headers[key] = value } } @@ -138,6 +139,9 @@ func (c *ComicClient) PrepareRequest(link, hostname string) (*http.Request, erro if err != nil { return nil, err } + if req == nil { + return nil, errors.New("request nil") + } if strings.Contains(hostname, "manganato") || strings.Contains(hostname, "mangakakalot") { req.Header.Set("Referer", link) diff --git a/pkg/sites/http_helpers.go b/pkg/sites/http_helpers.go index e9e29d8a..bf4a29ed 100644 --- a/pkg/sites/http_helpers.go +++ b/pkg/sites/http_helpers.go @@ -19,6 +19,35 @@ func defaultClient(client *httpclient.ComicClient) *httpclient.ComicClient { return httpclient.NewComicClient() } +type requestConfig struct { + request *http.Request +} + +func (rc *requestConfig) apply(opts []requestOption) { + for _, opt := range opts { + opt(rc) + } +} + +type requestOption func(*requestConfig) + +func withHttpHeader(key, value string) requestOption { + return func(rc *requestConfig) { + if rc.request == nil { + return + } + if rc.request.Header == nil { + rc.request.Header = make(http.Header) + } + // avoid overwriting existing headers, user may have set it in the client already + _, ok := rc.request.Header[key] + if ok { + return + } + rc.request.Header.Set(key, value) + } +} + func hostFromURL(link string) string { parsed, err := urlpkg.Parse(link) if err != nil { @@ -27,31 +56,37 @@ func hostFromURL(link string) string { return parsed.Host } -func buildRequest(ctx context.Context, client *httpclient.ComicClient, link string) (*http.Request, error) { +func buildRequest(ctx context.Context, client *httpclient.ComicClient, link string, opts ...requestOption) (*http.Request, error) { + reqConfig := &requestConfig{} req, err := client.PrepareRequest(link, hostFromURL(link)) if err != nil { return nil, err } + reqConfig.request = req + + // apply config to request + reqConfig.apply(opts) + if ctx != nil { req = req.WithContext(ctx) } return req, nil } -func fetchHTML(ctx context.Context, client *httpclient.ComicClient, link string) (string, error) { - response, err := fetchBytes(ctx, client, link) +func fetchHTML(ctx context.Context, client *httpclient.ComicClient, link string, opts ...requestOption) (string, error) { + response, err := fetchBytes(ctx, client, link, opts...) if err != nil { return "", err } return string(response), nil } -func fetchJSON(ctx context.Context, client *httpclient.ComicClient, link string, target interface{}) error { +func fetchJSON(ctx context.Context, client *httpclient.ComicClient, link string, target interface{}, opts ...requestOption) error { if target == nil { return fmt.Errorf("target cannot be nil") } - data, err := fetchBytes(ctx, client, link) + data, err := fetchBytes(ctx, client, link, opts...) if err != nil { return err } @@ -61,13 +96,13 @@ func fetchJSON(ctx context.Context, client *httpclient.ComicClient, link string, return json.Unmarshal(data, target) } -func fetchBytes(ctx context.Context, client *httpclient.ComicClient, link string) ([]byte, error) { +func fetchBytes(ctx context.Context, client *httpclient.ComicClient, link string, opts ...requestOption) ([]byte, error) { cc := defaultClient(client) if ctx == nil { ctx = context.Background() } - req, err := buildRequest(ctx, cc, link) + req, err := buildRequest(ctx, cc, link, opts...) if err != nil { return nil, err } From f216adc2d7159e053b3fe69608eb8c2f15395590 Mon Sep 17 00:00:00 2001 From: ProjectDislocate <173194962+ProjectDislocate@users.noreply.github.com> Date: Tue, 21 Apr 2026 16:45:28 -0400 Subject: [PATCH 45/79] feat: limit file name length --- pkg/util/path.go | 20 ++++++++++---- pkg/util/path_test.go | 62 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 77 insertions(+), 5 deletions(-) diff --git a/pkg/util/path.go b/pkg/util/path.go index 6d090f77..6f32029a 100644 --- a/pkg/util/path.go +++ b/pkg/util/path.go @@ -6,6 +6,8 @@ import ( "path/filepath" ) +const NameLength = 100 + // createPath create folders given the path. func createPath(path string) (string, error) { err := os.MkdirAll(path, os.ModePerm) @@ -21,11 +23,19 @@ func createPath(path string) (string, error) { return dir, err } +// TrimNameLength trims the name to a maximum length defined by NameLength constant +func trimNameLength(name string) string { + if len(name) > NameLength { + return name[:NameLength] + } + return name +} + // PathSetup creates the folders where the comic will be saved. // when `createDefaultPath` is false the comic is stored without prepending // the default folder path `comics/source/name/[comic.format]`. func PathSetup(createDefaultPath bool, outputFolder, source, name string) (string, error) { - path := fmt.Sprintf("%s/comics/%s/%s/", outputFolder, source, name) + path := fmt.Sprintf("%s/comics/%s/%s/", outputFolder, source, trimNameLength(name)) if !createDefaultPath { path = fmt.Sprintf("%s/", outputFolder) @@ -38,10 +48,10 @@ func PathSetup(createDefaultPath bool, outputFolder, source, name string) (strin // when `createDefaultPath` is false the images are stored without prepending // the default folder path `comics/source/name/[comic.format]`. func ImagesPathSetup(createDefaultPath bool, outputFolder, source, name, issueFolderName, issueNumber string) (string, error) { - path := fmt.Sprintf("%s/comics/%s/%s/images-%s/", outputFolder, source, name, issueNumber) + path := fmt.Sprintf("%s/comics/%s/%s/images-%s/", outputFolder, source, trimNameLength(name), trimNameLength(issueNumber)) if !createDefaultPath { - path = fmt.Sprintf("%s/%s%s", outputFolder, issueFolderName, issueNumber) + path = fmt.Sprintf("%s/%s", outputFolder, trimNameLength(issueFolderName+issueNumber)) } return createPath(path) @@ -66,7 +76,7 @@ func DirectoryOrFileDoesNotExist(filePath string) bool { // GetPathToFile returns the path where the file should be saved. func GetPathToFile(dir, name, issueNumber, format string, issueNumberOnly bool) string { if issueNumberOnly { - return fmt.Sprintf("%s/%s.%s", dir, issueNumber, format) + return fmt.Sprintf("%s/%s.%s", dir, trimNameLength(issueNumber), format) } - return fmt.Sprintf("%s/%s-%s.%s", dir, name, issueNumber, format) + return fmt.Sprintf("%s/%s-%s.%s", dir, trimNameLength(name), trimNameLength(issueNumber), format) } diff --git a/pkg/util/path_test.go b/pkg/util/path_test.go index b9b93c07..33ef8cee 100644 --- a/pkg/util/path_test.go +++ b/pkg/util/path_test.go @@ -1,8 +1,10 @@ package util import ( + "fmt" "os" "path/filepath" + "strings" "testing" "github.com/stretchr/testify/assert" @@ -32,3 +34,63 @@ func TestDirectoryOrFileDoesNotExist(t *testing.T) { assert.False(t, result) } + +func TestTrimNameLength(t *testing.T) { + shortName := "short-name" + exactLengthName := strings.Repeat("a", NameLength) + longName := strings.Repeat("b", NameLength+10) + + assert.Equal(t, shortName, trimNameLength(shortName)) + assert.Equal(t, exactLengthName, trimNameLength(exactLengthName)) + assert.Equal(t, strings.Repeat("b", NameLength), trimNameLength(longName)) +} + +func TestPathSetupTrimsComicName(t *testing.T) { + outputFolder := t.TempDir() + longComicName := strings.Repeat("comic", 30) + + result, err := PathSetup(true, outputFolder, "example-source", longComicName) + + assert.Nil(t, err) + assert.Contains(t, result, filepath.Join("comics", "example-source")) + assert.Contains(t, result, strings.Repeat("comic", 20)) + assert.NotContains(t, result, longComicName) +} + +func TestImagesPathSetupTrimsIssueNumber(t *testing.T) { + outputFolder := t.TempDir() + longIssueNumber := strings.Repeat("issue", 30) + + result, err := ImagesPathSetup(true, outputFolder, "source", "name", "issue-", longIssueNumber) + + assert.Nil(t, err) + assert.Contains(t, result, "images-") + assert.Contains(t, result, fmt.Sprintf("images-%s", strings.Repeat("issue", 20))) + assert.NotContains(t, result, longIssueNumber) +} + +func TestImagesPathSetupTrimsCustomFolderName(t *testing.T) { + outputFolder := t.TempDir() + longIssueFolderName := strings.Repeat("folder", 20) + longIssueNumber := strings.Repeat("number", 20) + + result, err := ImagesPathSetup(false, outputFolder, "source", "name", longIssueFolderName, longIssueNumber) + + assert.Nil(t, err) + assert.Equal(t, NameLength, len(filepath.Base(result))) + assert.NotContains(t, result, longIssueFolderName+longIssueNumber) +} + +func TestGetPathToFileTrimsNameAndIssueNumber(t *testing.T) { + dir := "path/to/something" + longName := strings.Repeat("n", NameLength+20) + longIssueNumber := strings.Repeat("i", NameLength+15) + + result := GetPathToFile(dir, longName, longIssueNumber, "pdf", false) + expected := fmt.Sprintf("%s/%s-%s.pdf", dir, strings.Repeat("n", NameLength), strings.Repeat("i", NameLength)) + assert.Equal(t, expected, result) + + resultIssueOnly := GetPathToFile(dir, longName, longIssueNumber, "pdf", true) + expectedIssueOnly := fmt.Sprintf("%s/%s.pdf", dir, strings.Repeat("i", NameLength)) + assert.Equal(t, expectedIssueOnly, resultIssueOnly) +} From ebdb8fe0876bf12eacfa8841e31744005bfe18d9 Mon Sep 17 00:00:00 2001 From: ProjectDislocate <173194962+ProjectDislocate@users.noreply.github.com> Date: Tue, 21 Apr 2026 16:46:00 -0400 Subject: [PATCH 46/79] fix: uninitiated SeriesMetadata fields --- pkg/sites/loader.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkg/sites/loader.go b/pkg/sites/loader.go index 1bb9a694..2b0e2cc5 100644 --- a/pkg/sites/loader.go +++ b/pkg/sites/loader.go @@ -73,7 +73,9 @@ func initializeCollection(issues []string, options *config.Options, base BaseSit URL: url, }, SeriesMetadata: &core.SeriesMetadata{ - Title: name, + Title: name, + LocalizedTitle: make(map[string]string), + Description: make(map[string]string), }, } options.Logger.Debugf("Initializing comic with URL: %s", comic.Source.URL) From 270e2725eb9fb73c9bf103f7c728385e77185e94 Mon Sep 17 00:00:00 2001 From: ProjectDislocate <173194962+ProjectDislocate@users.noreply.github.com> Date: Tue, 21 Apr 2026 16:47:03 -0400 Subject: [PATCH 47/79] feat: improved debug logging --- pkg/sites/loader.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pkg/sites/loader.go b/pkg/sites/loader.go index 2b0e2cc5..5ef38a7d 100644 --- a/pkg/sites/loader.go +++ b/pkg/sites/loader.go @@ -17,6 +17,8 @@ func initializeCollection(issues []string, options *config.Options, base BaseSit var collection []*core.ComicIssue // var err error + options.Logger.Debugf("sites: initializing collection for %d issue(s)", len(issues)) + if len(issues) == 0 { return collection, fmt.Errorf("no issues found for URL %q; ensure it points to a specific comic or chapter page", options.URL) } @@ -45,6 +47,7 @@ func initializeCollection(issues []string, options *config.Options, base BaseSit issueNumber = util.Parse(issueNumber) if notInIssuesRange(issueNumber, startRange, endRange) { + options.Logger.Debugf("Skipping issue %q as it is outside the specified range %q", issueNumber, options.IssuesRange) continue } @@ -60,6 +63,7 @@ func initializeCollection(issues []string, options *config.Options, base BaseSit fileName := util.GetPathToFile(dir, name, issueNumber, outputFormat.String(), options.IssueNumberNameOnly) if util.DirectoryOrFileDoesNotExist(fileName) || options.ImagesOnly { + options.Logger.Debugf("Adding issue %q to collection with URL: %s", issueNumber, url) comic := &core.ComicIssue{ Name: name, @@ -80,9 +84,12 @@ func initializeCollection(issues []string, options *config.Options, base BaseSit } options.Logger.Debugf("Initializing comic with URL: %s", comic.Source.URL) if err = base.Initialize(comic); err != nil { + options.Logger.Errorf("error initializing comic for url %q: %v", url, err) return collection, err } collection = append(collection, comic) + } else { + options.Logger.Debugf("Skipping issue %q as it already exists at path: %s", issueNumber, fileName) } } From 24c64c021b634ecfc1383dcc2ab2ffb1dd6e1341 Mon Sep 17 00:00:00 2001 From: ProjectDislocate <173194962+ProjectDislocate@users.noreply.github.com> Date: Tue, 21 Apr 2026 16:49:56 -0400 Subject: [PATCH 48/79] chore: bump deps --- go.mod | 16 +++++++--------- go.sum | 24 ++++++++++++------------ 2 files changed, 19 insertions(+), 21 deletions(-) diff --git a/go.mod b/go.mod index 77eeb95a..7fabb940 100644 --- a/go.mod +++ b/go.mod @@ -1,8 +1,6 @@ module github.com/Girbons/comics-downloader -go 1.23.0 - -toolchain go1.24.4 +go 1.25.0 require ( fyne.io/fyne v1.4.3 @@ -15,9 +13,9 @@ require ( github.com/schollz/progressbar/v2 v2.15.0 github.com/sirupsen/logrus v1.9.3 github.com/stretchr/testify v1.11.1 - golang.org/x/image v0.18.0 - golang.org/x/mod v0.17.0 - golang.org/x/sync v0.12.0 + golang.org/x/image v0.39.0 + golang.org/x/mod v0.35.0 + golang.org/x/sync v0.20.0 ) require ( @@ -54,8 +52,8 @@ require ( github.com/ulikunitz/xz v0.5.12 // indirect github.com/vincent-petithory/dataurl v0.0.0-20191104211930-d1553a71de50 // indirect go4.org v0.0.0-20230225012048-214862532bf5 // indirect - golang.org/x/net v0.38.0 // indirect - golang.org/x/sys v0.31.0 // indirect - golang.org/x/text v0.23.0 // indirect + golang.org/x/net v0.47.0 // indirect + golang.org/x/sys v0.38.0 // indirect + golang.org/x/text v0.36.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index c96321e6..9d31073d 100644 --- a/go.sum +++ b/go.sum @@ -211,8 +211,8 @@ golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMx golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/image v0.0.0-20190910094157-69e4b8554b2a/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/image v0.0.0-20200430140353-33d19683fad8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= -golang.org/x/image v0.18.0 h1:jGzIakQa/ZXI1I0Fxvaa9W7yP25TqT6cHIHn+6CqvSQ= -golang.org/x/image v0.18.0/go.mod h1:4yyo5vMFQjVjUcVk4jEQcU9MGy/rulF5WvUILseCM2E= +golang.org/x/image v0.39.0 h1:skVYidAEVKgn8lZ602XO75asgXBgLj9G/FE3RbuPFww= +golang.org/x/image v0.39.0/go.mod h1:sIbmppfU+xFLPIG0FoVUTvyBMmgng1/XAMhQ2ft0hpA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -229,8 +229,8 @@ golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= -golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.35.0 h1:Ww1D637e6Pg+Zb2KrWfHQUnH2dQRLBQyAtpr/haaJeM= +golang.org/x/mod v0.35.0/go.mod h1:+GwiRhIInF8wPm+4AoT6L0FA1QWAad3OMdTRx4tFYlU= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -251,8 +251,8 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210505024714-0287a6fb4125/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= -golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= +golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY= +golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -265,8 +265,8 @@ golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.12.0 h1:MHc5BpPuC30uJk597Ri8TV3CNZcTLu6B6z4lJy+g6Jw= -golang.org/x/sync v0.12.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= +golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -289,8 +289,8 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= -golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= +golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= @@ -302,8 +302,8 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= -golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= +golang.org/x/text v0.36.0 h1:JfKh3XmcRPqZPKevfXVpI1wXPTqbkE5f7JA92a55Yxg= +golang.org/x/text v0.36.0/go.mod h1:NIdBknypM8iqVmPiuco0Dh6P5Jcdk8lJL0CUebqK164= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= From 43df4034604323980d3e2e8fe5d2304adb3f57a8 Mon Sep 17 00:00:00 2001 From: ProjectDislocate <173194962+ProjectDislocate@users.noreply.github.com> Date: Tue, 21 Apr 2026 16:51:26 -0400 Subject: [PATCH 49/79] feat: dedupe probided urls --- cmd/app/downloader.go | 19 +++++++++++++------ pkg/util/util.go | 12 ++++++++++++ 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/cmd/app/downloader.go b/cmd/app/downloader.go index 63589366..b62db464 100644 --- a/cmd/app/downloader.go +++ b/cmd/app/downloader.go @@ -14,6 +14,7 @@ import ( "github.com/Girbons/comics-downloader/pkg/detector" httpclient "github.com/Girbons/comics-downloader/pkg/http" "github.com/Girbons/comics-downloader/pkg/sites" + "github.com/Girbons/comics-downloader/pkg/util" "github.com/sirupsen/logrus" ) @@ -138,17 +139,23 @@ func (r *Runner) download(base config.Options) { opts.Logger.Infof("A new comics-downloader version is available at %s", newVersionLink) } - for _, rawURL := range strings.Split(opts.URL, ",") { - trimmedURL := strings.TrimSpace(rawURL) - if trimmedURL == "" { + rawURLs := strings.Split(opts.URL, ",") + rawURLs = util.RemoveDuplicates(rawURLs) + if len(rawURLs) > 1 { + opts.Logger.Infof("Processing %d URLs", len(rawURLs)) + } + + for _, rawURL := range rawURLs { + cleanedURL := strings.TrimSpace(rawURL) + if cleanedURL == "" { continue } perURL := opts - perURL.URL = trimmedURL + perURL.URL = cleanedURL perURL.OutputFolder = outputFolder // check if the link is supported - source, isSupported, isDisabled := detector.DetectSource(trimmedURL) + source, isSupported, isDisabled := detector.DetectSource(cleanedURL) perURL.SourceName = source @@ -162,7 +169,7 @@ func (r *Runner) download(base config.Options) { continue } - perURL.Logger.Info("Downloading... " + trimmedURL) + perURL.Logger.Info("Downloading... " + cleanedURL) collection, err := sites.LoadComicFromSource(&perURL) if err != nil { perURL.Logger.Error(err.Error()) diff --git a/pkg/util/util.go b/pkg/util/util.go index a9ba9ced..8787e416 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -66,3 +66,15 @@ func Parse(s string) string { return strings.Trim(replacer.Replace(s), " ") } + +func RemoveDuplicates[T comparable](sliceList []T) []T { + allKeys := make(map[T]bool) + list := []T{} + for _, item := range sliceList { + if _, value := allKeys[item]; !value { + allKeys[item] = true + list = append(list, item) + } + } + return list +} From 8b3beb80bc7e28943aac8b12c6d7dc29dabed46a Mon Sep 17 00:00:00 2001 From: ProjectDislocate <173194962+ProjectDislocate@users.noreply.github.com> Date: Tue, 21 Apr 2026 16:54:49 -0400 Subject: [PATCH 50/79] feat: request caching --- cmd/app/downloader.go | 3 + cmd/app/runner_test.go | 13 ++++ cmd/downloader/main.go | 3 + cmd/downloader/main_test.go | 8 +++ pkg/config/options.go | 1 + pkg/http/cache.go | 126 +++++++++++++++++++++++++++++++++ pkg/http/cache_test.go | 76 ++++++++++++++++++++ pkg/http/client.go | 30 ++++++-- pkg/sites/http_helpers.go | 16 +++++ pkg/sites/http_helpers_test.go | 101 ++++++++++++++++++++++++++ 10 files changed, 370 insertions(+), 7 deletions(-) create mode 100644 pkg/http/cache.go create mode 100644 pkg/http/cache_test.go create mode 100644 pkg/sites/http_helpers_test.go diff --git a/cmd/app/downloader.go b/cmd/app/downloader.go index b62db464..63a49d29 100644 --- a/cmd/app/downloader.go +++ b/cmd/app/downloader.go @@ -209,6 +209,9 @@ func buildClientOptions(base config.Options) []httpclient.Option { defaultUA := fmt.Sprintf("comics-downloader/%s", version.Tag) agents := mergeUserAgents(defaultUA, base.UserAgents) opts := []httpclient.Option{httpclient.WithUserAgents(agents)} + if !base.NoCache { + opts = append(opts, httpclient.WithResponseCache(httpclient.NewInMemoryResponseCache(0))) + } if strings.TrimSpace(base.SessionCookie) != "" { opts = append(opts, httpclient.WithHeaders(map[string]string{ diff --git a/cmd/app/runner_test.go b/cmd/app/runner_test.go index 9797803d..3d1073eb 100644 --- a/cmd/app/runner_test.go +++ b/cmd/app/runner_test.go @@ -5,6 +5,7 @@ import ( "testing" "github.com/Girbons/comics-downloader/pkg/config" + httpclient "github.com/Girbons/comics-downloader/pkg/http" ) func TestRunnerPrepareOptionsProvidesDependencies(t *testing.T) { @@ -45,3 +46,15 @@ func TestRunnerRunRequiresURL(t *testing.T) { t.Fatalf("expected an error message to be sent") } } + +func TestBuildClientOptionsCacheToggle(t *testing.T) { + clientWithCache := httpclient.NewComicClient(buildClientOptions(config.Options{})...) + if clientWithCache.ResponseCache() == nil { + t.Fatalf("expected response cache to be enabled by default") + } + + clientWithoutCache := httpclient.NewComicClient(buildClientOptions(config.Options{NoCache: true})...) + if clientWithoutCache.ResponseCache() != nil { + t.Fatalf("expected response cache to be disabled when no-cache is set") + } +} diff --git a/cmd/downloader/main.go b/cmd/downloader/main.go index a4928cea..3e951124 100644 --- a/cmd/downloader/main.go +++ b/cmd/downloader/main.go @@ -47,6 +47,7 @@ var ( // request customization userAgentsCSV string sessionCookie string + noCache bool requestTimeout time.Duration // throttling requestDelay time.Duration @@ -73,6 +74,7 @@ func init() { flag.StringVar(&issueFolderName, "issue-folder-name", "issue-", "Folder name where each issue/chapter will be saved, default 'issue-#'") flag.StringVar(&userAgentsCSV, "user-agents", "", "Comma-separated list of alternative User-Agent values to rotate per request") flag.StringVar(&sessionCookie, "session-cookie", "", "Custom Cookie header value (e.g., cf_clearance=...; other=...) for protected sources") + flag.BoolVar(&noCache, "no-cache", false, "Disable in-memory metadata request caching") flag.DurationVar(&requestTimeout, "request-timeout", config.DefaulltRequestTimeout, "Timeout for HTTP requests (e.g., 8s)") flag.DurationVar(&requestDelay, "request-delay", config.DefaultRequestDelay, "Base delay inserted before downloading each image (e.g. 500ms)") flag.DurationVar(&requestDelayJitter, "request-delay-jitter", config.DefaultRequestDelayJitter, "Maximum additional random delay added to the base request delay (e.g. 250ms)") @@ -101,6 +103,7 @@ func buildOptions() config.Options { IssueFolderName: issueFolderName, UserAgents: splitAndTrim(userAgentsCSV), SessionCookie: strings.TrimSpace(sessionCookie), + NoCache: noCache, RequestTimeout: requestTimeout, RequestDelay: requestDelay, RequestDelayJitter: requestDelayJitter, diff --git a/cmd/downloader/main_test.go b/cmd/downloader/main_test.go index ae5744d9..8623eddf 100644 --- a/cmd/downloader/main_test.go +++ b/cmd/downloader/main_test.go @@ -25,6 +25,7 @@ func TestBuildOptionsCopiesGlobals(t *testing.T) { issueFolderName string userAgentsCSV string sessionCookie string + noCache bool }{ debug: debug, all: all, @@ -45,6 +46,7 @@ func TestBuildOptionsCopiesGlobals(t *testing.T) { issueFolderName: issueFolderName, userAgentsCSV: userAgentsCSV, sessionCookie: sessionCookie, + noCache: noCache, } defer func() { debug = prev.debug @@ -66,6 +68,7 @@ func TestBuildOptionsCopiesGlobals(t *testing.T) { issueFolderName = prev.issueFolderName userAgentsCSV = prev.userAgentsCSV sessionCookie = prev.sessionCookie + noCache = prev.noCache }() debug = true @@ -87,6 +90,7 @@ func TestBuildOptionsCopiesGlobals(t *testing.T) { issueFolderName = "chapter-" userAgentsCSV = "UA1, UA2 ," sessionCookie = "cf_clearance=abc123; other=value" + noCache = true opts := buildOptions() @@ -121,4 +125,8 @@ func TestBuildOptionsCopiesGlobals(t *testing.T) { if opts.SessionCookie != "cf_clearance=abc123; other=value" { t.Fatalf("expected session cookie to be copied, got %q", opts.SessionCookie) } + + if !opts.NoCache { + t.Fatalf("expected no-cache option to be copied, got %+v", opts) + } } diff --git a/pkg/config/options.go b/pkg/config/options.go index 52ecbf0d..869a04d7 100644 --- a/pkg/config/options.go +++ b/pkg/config/options.go @@ -39,6 +39,7 @@ type Options struct { UserAgents []string SessionCookie string + NoCache bool RequestDelay time.Duration RequestDelayJitter time.Duration RequestTimeout time.Duration diff --git a/pkg/http/cache.go b/pkg/http/cache.go new file mode 100644 index 00000000..fb9a9bb5 --- /dev/null +++ b/pkg/http/cache.go @@ -0,0 +1,126 @@ +package http + +import ( + "crypto/sha256" + "encoding/hex" + "net/http" + "sort" + "strings" + "sync" +) + +const defaultResponseCacheCapacity = 2048 + +// ResponseCache stores response bodies for metadata requests. +type ResponseCache interface { + Get(key string) ([]byte, bool) + Set(key string, value []byte) + Clear() +} + +// InMemoryResponseCache is a bounded in-memory cache implementation. +type InMemoryResponseCache struct { + mu sync.RWMutex + maxItems int + items map[string][]byte + order []string +} + +// NewInMemoryResponseCache creates a process-local in-memory response cache. +func NewInMemoryResponseCache(maxItems int) *InMemoryResponseCache { + if maxItems <= 0 { + maxItems = defaultResponseCacheCapacity + } + + return &InMemoryResponseCache{ + maxItems: maxItems, + items: make(map[string][]byte), + order: make([]string, 0, maxItems), + } +} + +// Get returns a copy of the cached value when present. +func (c *InMemoryResponseCache) Get(key string) ([]byte, bool) { + if c == nil || key == "" { + return nil, false + } + + c.mu.RLock() + value, ok := c.items[key] + c.mu.RUnlock() + if !ok { + return nil, false + } + + out := make([]byte, len(value)) + copy(out, value) + return out, true +} + +// Set stores a copy of the value in the cache. +func (c *InMemoryResponseCache) Set(key string, value []byte) { + if c == nil || key == "" || len(value) == 0 { + return + } + + cached := make([]byte, len(value)) + copy(cached, value) + + c.mu.Lock() + defer c.mu.Unlock() + + if _, exists := c.items[key]; !exists { + c.order = append(c.order, key) + } + c.items[key] = cached + + for len(c.order) > c.maxItems { + oldest := c.order[0] + c.order = c.order[1:] + delete(c.items, oldest) + } +} + +// Clear removes all cache entries. +func (c *InMemoryResponseCache) Clear() { + if c == nil { + return + } + + c.mu.Lock() + defer c.mu.Unlock() + + c.items = make(map[string][]byte) + c.order = c.order[:0] +} + +// MetadataCacheKey builds a deterministic cache key for metadata GET requests. +func MetadataCacheKey(req *http.Request) string { + if req == nil || req.URL == nil { + return "" + } + + builder := strings.Builder{} + builder.WriteString(req.Method) + builder.WriteString("\n") + builder.WriteString(req.URL.String()) + builder.WriteString("\n") + + headers := []string{"Accept", "Cookie", "Referer", "User-Agent"} + sort.Strings(headers) + for _, key := range headers { + values := req.Header.Values(key) + if len(values) == 0 { + continue + } + copied := append([]string(nil), values...) + sort.Strings(copied) + builder.WriteString(strings.ToLower(key)) + builder.WriteString(":") + builder.WriteString(strings.Join(copied, ",")) + builder.WriteString("\n") + } + + sum := sha256.Sum256([]byte(builder.String())) + return hex.EncodeToString(sum[:]) +} diff --git a/pkg/http/cache_test.go b/pkg/http/cache_test.go new file mode 100644 index 00000000..767537db --- /dev/null +++ b/pkg/http/cache_test.go @@ -0,0 +1,76 @@ +package http + +import ( + "net/http" + "testing" +) + +func TestMetadataCacheKeyDeterministic(t *testing.T) { + reqA, err := http.NewRequest(http.MethodGet, "https://example.com/a?x=1", nil) + if err != nil { + t.Fatalf("failed creating request A: %v", err) + } + reqA.Header.Set("Accept", "text/html") + reqA.Header.Set("User-Agent", "UA-1") + reqA.Header.Set("Cookie", "cf=abc") + + reqB, err := http.NewRequest(http.MethodGet, "https://example.com/a?x=1", nil) + if err != nil { + t.Fatalf("failed creating request B: %v", err) + } + reqB.Header.Set("Cookie", "cf=abc") + reqB.Header.Set("User-Agent", "UA-1") + reqB.Header.Set("Accept", "text/html") + + keyA := MetadataCacheKey(reqA) + keyB := MetadataCacheKey(reqB) + if keyA == "" || keyB == "" { + t.Fatalf("expected non-empty keys") + } + if keyA != keyB { + t.Fatalf("expected deterministic keys, got %q and %q", keyA, keyB) + } +} + +func TestMetadataCacheKeyVariesByHeaders(t *testing.T) { + reqA, _ := http.NewRequest(http.MethodGet, "https://example.com/a", nil) + reqB, _ := http.NewRequest(http.MethodGet, "https://example.com/a", nil) + reqA.Header.Set("Accept", "text/html") + reqB.Header.Set("Accept", "application/json") + + if MetadataCacheKey(reqA) == MetadataCacheKey(reqB) { + t.Fatalf("expected cache key to differ when request headers differ") + } +} + +func TestInMemoryResponseCacheStoresCopiesAndEvicts(t *testing.T) { + cache := NewInMemoryResponseCache(1) + original := []byte("value-1") + cache.Set("a", original) + original[0] = 'X' + + cached, ok := cache.Get("a") + if !ok { + t.Fatalf("expected cached value") + } + if string(cached) != "value-1" { + t.Fatalf("expected cached copy, got %q", string(cached)) + } + + cached[0] = 'Y' + reloaded, ok := cache.Get("a") + if !ok { + t.Fatalf("expected cached value on second get") + } + if string(reloaded) != "value-1" { + t.Fatalf("expected cache to return immutable copy, got %q", string(reloaded)) + } + + cache.Set("b", []byte("value-2")) + if _, ok := cache.Get("a"); ok { + t.Fatalf("expected oldest key to be evicted") + } + if _, ok := cache.Get("b"); !ok { + t.Fatalf("expected newest key to remain") + } +} diff --git a/pkg/http/client.go b/pkg/http/client.go index 91767698..62add362 100644 --- a/pkg/http/client.go +++ b/pkg/http/client.go @@ -53,6 +53,13 @@ func WithRateLimiter(limiter RateLimiter) Option { } } +// WithResponseCache sets a response cache for metadata requests. +func WithResponseCache(cache ResponseCache) Option { + return func(cc *ComicClient) { + cc.responseCache = cache + } +} + // WithUserAgent overrides the default user-agent header with a single value. func WithUserAgent(agent string) Option { return WithUserAgents([]string{agent}) @@ -89,13 +96,14 @@ func WithHeaders(headers map[string]string) Option { // ComicClient is the custom HTTP helper used across the downloader. type ComicClient struct { - client *http.Client - retryCount int - retryWait time.Duration - rateLimiter RateLimiter - userAgents []string - headers map[string]string - uaCounter uint32 + client *http.Client + retryCount int + retryWait time.Duration + rateLimiter RateLimiter + responseCache ResponseCache + userAgents []string + headers map[string]string + uaCounter uint32 } // NewComicClient returns a ComicClient instance with sane defaults. @@ -133,6 +141,14 @@ func (c *ComicClient) HTTPClient() *http.Client { return c.client } +// ResponseCache exposes the configured metadata response cache. +func (c *ComicClient) ResponseCache() ResponseCache { + if c == nil { + return nil + } + return c.responseCache +} + // PrepareRequest setup a `GET` request with custom headers. func (c *ComicClient) PrepareRequest(link, hostname string) (*http.Request, error) { req, err := http.NewRequest(http.MethodGet, link, nil) diff --git a/pkg/sites/http_helpers.go b/pkg/sites/http_helpers.go index bf4a29ed..678b7dfe 100644 --- a/pkg/sites/http_helpers.go +++ b/pkg/sites/http_helpers.go @@ -107,6 +107,18 @@ func fetchBytes(ctx context.Context, client *httpclient.ComicClient, link string return nil, err } + cache := cc.ResponseCache() + cacheKey := "" + if cache != nil { + cacheKey = httpclient.MetadataCacheKey(req) + if cacheKey != "" { + if cached, ok := cache.Get(cacheKey); ok { + // log.Printf("Cache hit for request %s (%s)\n", req.URL.String(), cacheKey) + return cached, nil + } + } + } + resp, err := cc.Do(req) if err != nil { return nil, err @@ -125,5 +137,9 @@ func fetchBytes(ctx context.Context, client *httpclient.ComicClient, link string if err != nil { return nil, err } + + if cache != nil && cacheKey != "" { + cache.Set(cacheKey, body) + } return body, nil } diff --git a/pkg/sites/http_helpers_test.go b/pkg/sites/http_helpers_test.go new file mode 100644 index 00000000..20fa5696 --- /dev/null +++ b/pkg/sites/http_helpers_test.go @@ -0,0 +1,101 @@ +package sites + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + + httpclient "github.com/Girbons/comics-downloader/pkg/http" +) + +func TestFetchHTMLUsesMetadataCache(t *testing.T) { + var hits int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + atomic.AddInt32(&hits, 1) + fmt.Fprint(w, "ok") + })) + defer server.Close() + + client := httpclient.NewComicClient( + httpclient.WithHTTPClient(server.Client()), + httpclient.WithRetry(0, 0), + httpclient.WithResponseCache(httpclient.NewInMemoryResponseCache(64)), + ) + + ctx := context.Background() + first, err := fetchHTML(ctx, client, server.URL+"/metadata") + if err != nil { + t.Fatalf("first fetch failed: %v", err) + } + second, err := fetchHTML(ctx, client, server.URL+"/metadata") + if err != nil { + t.Fatalf("second fetch failed: %v", err) + } + if first != second { + t.Fatalf("expected identical response bodies") + } + if got := atomic.LoadInt32(&hits); got != 1 { + t.Fatalf("expected one upstream hit, got %d", got) + } +} + +func TestFetchHTMLCacheKeyIncludesRequestHeaders(t *testing.T) { + var hits int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + atomic.AddInt32(&hits, 1) + fmt.Fprint(w, "ok") + })) + defer server.Close() + + client := httpclient.NewComicClient( + httpclient.WithHTTPClient(server.Client()), + httpclient.WithRetry(0, 0), + httpclient.WithResponseCache(httpclient.NewInMemoryResponseCache(64)), + ) + + ctx := context.Background() + _, err := fetchHTML(ctx, client, server.URL+"/metadata", withHttpHeader("Accept", "text/html")) + if err != nil { + t.Fatalf("first fetch failed: %v", err) + } + _, err = fetchHTML(ctx, client, server.URL+"/metadata", withHttpHeader("Accept", "application/json")) + if err != nil { + t.Fatalf("second fetch failed: %v", err) + } + + if got := atomic.LoadInt32(&hits); got != 2 { + t.Fatalf("expected two upstream hits for distinct headers, got %d", got) + } +} + +func TestFetchHTMLDoesNotCacheErrorResponses(t *testing.T) { + var hits int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + atomic.AddInt32(&hits, 1) + w.WriteHeader(http.StatusInternalServerError) + })) + defer server.Close() + + client := httpclient.NewComicClient( + httpclient.WithHTTPClient(server.Client()), + httpclient.WithRetry(0, 0), + httpclient.WithResponseCache(httpclient.NewInMemoryResponseCache(64)), + ) + + ctx := context.Background() + _, err := fetchHTML(ctx, client, server.URL+"/metadata") + if err == nil { + t.Fatalf("expected first fetch to fail") + } + _, err = fetchHTML(ctx, client, server.URL+"/metadata") + if err == nil { + t.Fatalf("expected second fetch to fail") + } + + if got := atomic.LoadInt32(&hits); got != 2 { + t.Fatalf("expected two upstream hits because errors are not cached, got %d", got) + } +} From 9e36ee4d7dbe3ed246f17aacc35c94cce532f04f Mon Sep 17 00:00:00 2001 From: ProjectDislocate <173194962+ProjectDislocate@users.noreply.github.com> Date: Tue, 21 Apr 2026 16:59:42 -0400 Subject: [PATCH 51/79] fix: apparently web links are comma seperated --- pkg/core/output.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/core/output.go b/pkg/core/output.go index 1f495fb6..4ede8d69 100644 --- a/pkg/core/output.go +++ b/pkg/core/output.go @@ -133,7 +133,7 @@ func (comic *ComicIssue) makeComicInfoXML(options *config.Options, images *Downl comicInfo.CreateElement("Genre").SetText(genres) } if len(comic.SeriesMetadata.WebLinks) > 0 { - comicInfo.CreateElement("Web").SetText(strings.Join(comic.SeriesMetadata.WebLinks, " ")) + comicInfo.CreateElement("Web").SetText(strings.Join(comic.SeriesMetadata.WebLinks, ",")) } if len(comic.SeriesMetadata.Creators) > 0 { var writers []string From e2062cc7454cec8c25f2da7159541f9291aac2b1 Mon Sep 17 00:00:00 2001 From: ProjectDislocate <173194962+ProjectDislocate@users.noreply.github.com> Date: Tue, 21 Apr 2026 17:01:33 -0400 Subject: [PATCH 52/79] refactor: move http helpers to http package --- pkg/core/core.go | 2 +- pkg/http/client.go | 7 +- pkg/http/client_test.go | 2 +- pkg/http/http_helpers.go | 141 ++++++++++++++++++++++ pkg/{sites => http}/http_helpers_test.go | 40 +++---- pkg/sites/comicextra.go | 6 +- pkg/sites/comicextra_test.go | 7 ++ pkg/sites/common.go | 6 +- pkg/sites/http_helpers.go | 145 ----------------------- pkg/sites/mangadex.go | 14 +-- pkg/sites/manganato_test.go | 7 ++ pkg/sites/mangareader.go | 6 +- pkg/sites/mangareader_test.go | 7 ++ pkg/sites/mangatown.go | 8 +- pkg/sites/mangatown_test.go | 7 ++ pkg/sites/readallcomics.go | 4 +- pkg/sites/readallcomics_test.go | 7 ++ pkg/sites/readcomiconline.go | 6 +- pkg/sites/readcomiconline_test.go | 7 ++ 19 files changed, 233 insertions(+), 196 deletions(-) create mode 100644 pkg/http/http_helpers.go rename pkg/{sites => http}/http_helpers_test.go (62%) delete mode 100644 pkg/sites/http_helpers.go diff --git a/pkg/core/core.go b/pkg/core/core.go index f7f8ee81..4ce0a9c8 100644 --- a/pkg/core/core.go +++ b/pkg/core/core.go @@ -337,7 +337,7 @@ func (comic *ComicIssue) DownloadImages(options *config.Options) (*DownloadResul } request = request.WithContext(reqCtx) - response, err := client.Do(request) + response, err := client.DoRaw(request) if err != nil { return err } diff --git a/pkg/http/client.go b/pkg/http/client.go index 62add362..060e57f4 100644 --- a/pkg/http/client.go +++ b/pkg/http/client.go @@ -176,8 +176,9 @@ func (c *ComicClient) PrepareRequest(link, hostname string) (*http.Request, erro return req, nil } -// Do executes an HTTP request applying retry, timeout, and rate limiting policies. -func (c *ComicClient) Do(req *http.Request) (*http.Response, error) { +// DoRaw executes an HTTP request applying retry, timeout, and rate limiting policies. +// It is recommended to use the higher-level FetchHTML or FetchJSON methods instead of DoRaw for automatic response handling. +func (c *ComicClient) DoRaw(req *http.Request) (*http.Response, error) { if c == nil { return nil, errors.New("comic client is nil") } @@ -236,7 +237,7 @@ func (c *ComicClient) Get(link, hostname string) (*http.Response, error) { return nil, err } - return c.Do(request) + return c.DoRaw(request) } func (c *ComicClient) wait(ctx context.Context) error { diff --git a/pkg/http/client_test.go b/pkg/http/client_test.go index 067fd784..b76ee2e9 100644 --- a/pkg/http/client_test.go +++ b/pkg/http/client_test.go @@ -84,7 +84,7 @@ func TestRateLimiterInvoked(t *testing.T) { require.NoError(t, err) req = req.WithContext(ctx) - _, err = client.Do(req) + _, err = client.DoRaw(req) require.NoError(t, err) require.Equal(t, int32(1), atomic.LoadInt32(&limiter.count)) } diff --git a/pkg/http/http_helpers.go b/pkg/http/http_helpers.go new file mode 100644 index 00000000..1b639855 --- /dev/null +++ b/pkg/http/http_helpers.go @@ -0,0 +1,141 @@ +package http + +import ( + "context" + "encoding/json" + "fmt" + "io" + "log" + "net/http" + urlpkg "net/url" +) + +type RequestConfig struct { + request *http.Request +} + +func (rc *RequestConfig) apply(opts []RequestOption) { + for _, opt := range opts { + opt(rc) + } +} + +type RequestOption func(*RequestConfig) + +// withHttpHeader adds a header to the request, but only if it doesn't already exist. This allows users to set headers at the client level and override them on a per-request basis without worrying about conflicts. +func WithHttpHeader(key, value string) RequestOption { + return func(rc *RequestConfig) { + if rc.request == nil { + return + } + if rc.request.Header == nil { + rc.request.Header = make(http.Header) + } + // avoid overwriting existing headers, user may have set it in the client already + _, ok := rc.request.Header[key] + if ok { + return + } + rc.request.Header.Set(key, value) + } +} + +// hostFromURL extracts the hostname from a given URL string. If the URL is invalid, it returns an empty string. +func hostFromURL(link string) string { + parsed, err := urlpkg.Parse(link) + if err != nil { + return "" + } + return parsed.Host +} + +// buildRequest constructs an HTTP request based on the provided link and options. It applies any request options to the request configuration and attaches the context to the request if provided. +func (c *ComicClient) buildRequest(ctx context.Context, link string, opts ...RequestOption) (*http.Request, error) { + reqConfig := &RequestConfig{} + req, err := c.PrepareRequest(link, hostFromURL(link)) + if err != nil { + return nil, err + } + reqConfig.request = req + + // apply config to request + reqConfig.apply(opts) + + if ctx != nil { + req = req.WithContext(ctx) + } + return req, nil +} + +// FetchHTML retrieves the HTML content from the specified link with retry, timeout, rate limiting, and caching policies applied. +func (c *ComicClient) FetchHTML(ctx context.Context, link string, opts ...RequestOption) (string, error) { + response, err := c.FetchBytes(ctx, link, opts...) + if err != nil { + return "", err + } + return string(response), nil +} + +// FetchJSON retrieves the JSON content from the specified link, applies retry, timeout, rate limiting, and caching policies, and unmarshals the response into the provided target structure. +func (c *ComicClient) FetchJSON(ctx context.Context, link string, target interface{}, opts ...RequestOption) error { + if target == nil { + return fmt.Errorf("target cannot be nil") + } + + data, err := c.FetchBytes(ctx, link, opts...) + if err != nil { + return err + } + if len(data) == 0 { + return fmt.Errorf("empty response for %s", link) + } + return json.Unmarshal(data, target) +} + +// FetchBytes retrieves the raw byte content from the specified link, applying retry, timeout, rate limiting, and caching policies. It returns the response body as a byte slice. +func (c *ComicClient) FetchBytes(ctx context.Context, link string, opts ...RequestOption) ([]byte, error) { + if ctx == nil { + ctx = context.Background() + } + + req, err := c.buildRequest(ctx, link, opts...) + if err != nil { + return nil, err + } + + cache := c.ResponseCache() + cacheKey := "" + if cache != nil { + cacheKey = MetadataCacheKey(req) + if cacheKey != "" { + if cached, ok := cache.Get(cacheKey); ok { + // log.Printf("Cache hit for request %s (%s)\n", req.URL.String(), cacheKey) + return cached, nil + } + } + } + + resp, err := c.DoRaw(req) + if err != nil { + return nil, err + } + defer func() { + if closeErr := resp.Body.Close(); closeErr != nil { + log.Printf("sites: failed to close response body for %s: %v", link, closeErr) + } + }() + + if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices { + return nil, fmt.Errorf("unexpected status code %d for %s", resp.StatusCode, link) + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + if cache != nil && cacheKey != "" { + cache.Set(cacheKey, body) + } + return body, nil +} diff --git a/pkg/sites/http_helpers_test.go b/pkg/http/http_helpers_test.go similarity index 62% rename from pkg/sites/http_helpers_test.go rename to pkg/http/http_helpers_test.go index 20fa5696..08d3a3c3 100644 --- a/pkg/sites/http_helpers_test.go +++ b/pkg/http/http_helpers_test.go @@ -1,4 +1,4 @@ -package sites +package http import ( "context" @@ -7,8 +7,6 @@ import ( "net/http/httptest" "sync/atomic" "testing" - - httpclient "github.com/Girbons/comics-downloader/pkg/http" ) func TestFetchHTMLUsesMetadataCache(t *testing.T) { @@ -19,18 +17,18 @@ func TestFetchHTMLUsesMetadataCache(t *testing.T) { })) defer server.Close() - client := httpclient.NewComicClient( - httpclient.WithHTTPClient(server.Client()), - httpclient.WithRetry(0, 0), - httpclient.WithResponseCache(httpclient.NewInMemoryResponseCache(64)), + client := NewComicClient( + WithHTTPClient(server.Client()), + WithRetry(0, 0), + WithResponseCache(NewInMemoryResponseCache(64)), ) ctx := context.Background() - first, err := fetchHTML(ctx, client, server.URL+"/metadata") + first, err := client.FetchHTML(ctx, server.URL+"/metadata") if err != nil { t.Fatalf("first fetch failed: %v", err) } - second, err := fetchHTML(ctx, client, server.URL+"/metadata") + second, err := client.FetchHTML(ctx, server.URL+"/metadata") if err != nil { t.Fatalf("second fetch failed: %v", err) } @@ -50,18 +48,18 @@ func TestFetchHTMLCacheKeyIncludesRequestHeaders(t *testing.T) { })) defer server.Close() - client := httpclient.NewComicClient( - httpclient.WithHTTPClient(server.Client()), - httpclient.WithRetry(0, 0), - httpclient.WithResponseCache(httpclient.NewInMemoryResponseCache(64)), + client := NewComicClient( + WithHTTPClient(server.Client()), + WithRetry(0, 0), + WithResponseCache(NewInMemoryResponseCache(64)), ) ctx := context.Background() - _, err := fetchHTML(ctx, client, server.URL+"/metadata", withHttpHeader("Accept", "text/html")) + _, err := client.FetchHTML(ctx, server.URL+"/metadata", WithHttpHeader("Accept", "text/html")) if err != nil { t.Fatalf("first fetch failed: %v", err) } - _, err = fetchHTML(ctx, client, server.URL+"/metadata", withHttpHeader("Accept", "application/json")) + _, err = client.FetchHTML(ctx, server.URL+"/metadata", WithHttpHeader("Accept", "application/json")) if err != nil { t.Fatalf("second fetch failed: %v", err) } @@ -79,18 +77,18 @@ func TestFetchHTMLDoesNotCacheErrorResponses(t *testing.T) { })) defer server.Close() - client := httpclient.NewComicClient( - httpclient.WithHTTPClient(server.Client()), - httpclient.WithRetry(0, 0), - httpclient.WithResponseCache(httpclient.NewInMemoryResponseCache(64)), + client := NewComicClient( + WithHTTPClient(server.Client()), + WithRetry(0, 0), + WithResponseCache(NewInMemoryResponseCache(64)), ) ctx := context.Background() - _, err := fetchHTML(ctx, client, server.URL+"/metadata") + _, err := client.FetchHTML(ctx, server.URL+"/metadata") if err == nil { t.Fatalf("expected first fetch to fail") } - _, err = fetchHTML(ctx, client, server.URL+"/metadata") + _, err = client.FetchHTML(ctx, server.URL+"/metadata") if err == nil { t.Fatalf("expected second fetch to fail") } diff --git a/pkg/sites/comicextra.go b/pkg/sites/comicextra.go index e76558b0..5c9f6635 100644 --- a/pkg/sites/comicextra.go +++ b/pkg/sites/comicextra.go @@ -36,7 +36,7 @@ func (c *Comicextra) retrieveImageLinks(comic *core.ComicIssue) ([]string, error ctx, cancel := c.requestContext() defer cancel() - response, err := fetchHTML(ctx, c.client, comic.Source.URL) + response, err := c.client.FetchHTML(ctx, comic.Source.URL) if err != nil { return nil, err } @@ -67,7 +67,7 @@ func (c *Comicextra) retrieveLastIssue(url string) (string, error) { ctx, cancel := c.requestContext() defer cancel() - response, err := fetchHTML(ctx, c.client, url) + response, err := c.client.FetchHTML(ctx, url) if err != nil { return "", err } @@ -131,7 +131,7 @@ func (c *Comicextra) RetrieveIssueLinks() ([]string, error) { ctx, cancel := c.requestContext() defer cancel() - response, err := fetchHTML(ctx, c.client, url) + response, err := c.client.FetchHTML(ctx, url) if err != nil { return nil, err } diff --git a/pkg/sites/comicextra_test.go b/pkg/sites/comicextra_test.go index b058e134..fdc396af 100644 --- a/pkg/sites/comicextra_test.go +++ b/pkg/sites/comicextra_test.go @@ -8,6 +8,7 @@ import ( "github.com/Girbons/comics-downloader/internal/logger" "github.com/Girbons/comics-downloader/pkg/config" "github.com/Girbons/comics-downloader/pkg/core" + httpclient "github.com/Girbons/comics-downloader/pkg/http" "github.com/stretchr/testify/require" ) @@ -62,10 +63,16 @@ func TestComicExtraScraper(t *testing.T) { server := newComicExtraServer() defer server.Close() + client := httpclient.NewComicClient( + httpclient.WithHTTPClient(server.Client()), + httpclient.WithRetry(0, 0), + ) + opts := &config.Options{ URL: server.URL + comicExtraIssueFullPath, Logger: logger.NewLogger(false, nil), RequestTimeout: config.DefaulltRequestTimeout, + Client: client, } comicextra := NewComicextra(opts) diff --git a/pkg/sites/common.go b/pkg/sites/common.go index 39711260..90574d2f 100644 --- a/pkg/sites/common.go +++ b/pkg/sites/common.go @@ -21,7 +21,7 @@ func MangaKakalotGetInfo(options *config.Options, domain string, url string) (na ctx, cancel := mangaKakalotRequestContext(options) defer cancel() - res, err := fetchHTML(ctx, options.Client, url) + res, err := options.Client.FetchHTML(ctx, url) if err != nil { return "", "", err } @@ -54,7 +54,7 @@ func MangaKakalotInitialize(options *config.Options, comic *core.ComicIssue) err ctx, cancel := mangaKakalotRequestContext(options) defer cancel() - res, err := fetchHTML(ctx, options.Client, comic.Source.URL) + res, err := options.Client.FetchHTML(ctx, comic.Source.URL) if err != nil { return err } @@ -78,7 +78,7 @@ func MangaKakalotRetrieveIssueLinks(options *config.Options, domain string, url ctx, cancel := mangaKakalotRequestContext(options) defer cancel() - res, err := fetchHTML(ctx, options.Client, url) + res, err := options.Client.FetchHTML(ctx, url) if err != nil { return nil, err } diff --git a/pkg/sites/http_helpers.go b/pkg/sites/http_helpers.go deleted file mode 100644 index 678b7dfe..00000000 --- a/pkg/sites/http_helpers.go +++ /dev/null @@ -1,145 +0,0 @@ -package sites - -import ( - "context" - "encoding/json" - "fmt" - "io" - "log" - "net/http" - urlpkg "net/url" - - httpclient "github.com/Girbons/comics-downloader/pkg/http" -) - -func defaultClient(client *httpclient.ComicClient) *httpclient.ComicClient { - if client != nil { - return client - } - return httpclient.NewComicClient() -} - -type requestConfig struct { - request *http.Request -} - -func (rc *requestConfig) apply(opts []requestOption) { - for _, opt := range opts { - opt(rc) - } -} - -type requestOption func(*requestConfig) - -func withHttpHeader(key, value string) requestOption { - return func(rc *requestConfig) { - if rc.request == nil { - return - } - if rc.request.Header == nil { - rc.request.Header = make(http.Header) - } - // avoid overwriting existing headers, user may have set it in the client already - _, ok := rc.request.Header[key] - if ok { - return - } - rc.request.Header.Set(key, value) - } -} - -func hostFromURL(link string) string { - parsed, err := urlpkg.Parse(link) - if err != nil { - return "" - } - return parsed.Host -} - -func buildRequest(ctx context.Context, client *httpclient.ComicClient, link string, opts ...requestOption) (*http.Request, error) { - reqConfig := &requestConfig{} - req, err := client.PrepareRequest(link, hostFromURL(link)) - if err != nil { - return nil, err - } - reqConfig.request = req - - // apply config to request - reqConfig.apply(opts) - - if ctx != nil { - req = req.WithContext(ctx) - } - return req, nil -} - -func fetchHTML(ctx context.Context, client *httpclient.ComicClient, link string, opts ...requestOption) (string, error) { - response, err := fetchBytes(ctx, client, link, opts...) - if err != nil { - return "", err - } - return string(response), nil -} - -func fetchJSON(ctx context.Context, client *httpclient.ComicClient, link string, target interface{}, opts ...requestOption) error { - if target == nil { - return fmt.Errorf("target cannot be nil") - } - - data, err := fetchBytes(ctx, client, link, opts...) - if err != nil { - return err - } - if len(data) == 0 { - return fmt.Errorf("empty response for %s", link) - } - return json.Unmarshal(data, target) -} - -func fetchBytes(ctx context.Context, client *httpclient.ComicClient, link string, opts ...requestOption) ([]byte, error) { - cc := defaultClient(client) - if ctx == nil { - ctx = context.Background() - } - - req, err := buildRequest(ctx, cc, link, opts...) - if err != nil { - return nil, err - } - - cache := cc.ResponseCache() - cacheKey := "" - if cache != nil { - cacheKey = httpclient.MetadataCacheKey(req) - if cacheKey != "" { - if cached, ok := cache.Get(cacheKey); ok { - // log.Printf("Cache hit for request %s (%s)\n", req.URL.String(), cacheKey) - return cached, nil - } - } - } - - resp, err := cc.Do(req) - if err != nil { - return nil, err - } - defer func() { - if closeErr := resp.Body.Close(); closeErr != nil { - log.Printf("sites: failed to close response body for %s: %v", link, closeErr) - } - }() - - if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices { - return nil, fmt.Errorf("unexpected status code %d for %s", resp.StatusCode, link) - } - - body, err := io.ReadAll(resp.Body) - if err != nil { - return nil, err - } - - if cache != nil && cacheKey != "" { - cache.Set(cacheKey, body) - } - return body, nil -} diff --git a/pkg/sites/mangadex.go b/pkg/sites/mangadex.go index dcfb3249..aa2aebe9 100644 --- a/pkg/sites/mangadex.go +++ b/pkg/sites/mangadex.go @@ -95,7 +95,7 @@ func (m *Mangadex) getMangaCoverURL(mangaID, coverID string) (string, error) { } `json:"data"` } - if err := fetchJSON(ctx, m.client, endpoint, &coverRes); err != nil { + if err := m.client.FetchJSON(ctx, endpoint, &coverRes); err != nil { return "", err } if strings.ToLower(coverRes.Result) != "ok" { @@ -119,7 +119,7 @@ func (m *Mangadex) getAuthorInfo(authorID string) (string, error) { } `json:"data"` } - if err := fetchJSON(ctx, m.client, endpoint, &authorRes); err != nil { + if err := m.client.FetchJSON(ctx, endpoint, &authorRes); err != nil { return "", err } if strings.ToLower(authorRes.Result) != "ok" { @@ -143,7 +143,7 @@ func (m *Mangadex) getMangaRating(mangaID string) (float64, error) { } `json:"rating"` } `json:"statistics"` } - if err := fetchJSON(ctx, m.client, endpoint, &ratingRes); err != nil { + if err := m.client.FetchJSON(ctx, endpoint, &ratingRes); err != nil { return 0, err } if strings.ToLower(ratingRes.Result) != "ok" { @@ -195,7 +195,7 @@ func (m *Mangadex) getMangaInfo(mangaID string) (mangadexSeries, error) { } `json:"data"` } - if err := fetchJSON(ctx, m.client, endpoint, &mangaRes); err != nil { + if err := m.client.FetchJSON(ctx, endpoint, &mangaRes); err != nil { return mangadexSeries{}, err } if strings.ToLower(mangaRes.Result) != "ok" { @@ -398,7 +398,7 @@ func (m *Mangadex) getChapters(mangaID string) ([]string, error) { endpoint += "?" + q.Encode() } - body, err := fetchBytes(ctx, m.client, endpoint) + body, err := m.client.FetchBytes(ctx, endpoint) if err != nil { return nil, err } @@ -484,7 +484,7 @@ func (m *Mangadex) getChapterInfo(chapterID string) (chapterInfo mangadexChapter } `json:"data"` } - if err := fetchJSON(ctx, m.client, endpoint, &chapterRes); err != nil { + if err := m.client.FetchJSON(ctx, endpoint, &chapterRes); err != nil { return mangadexChapter{}, err } if strings.ToLower(chapterRes.Result) != "ok" { @@ -505,7 +505,7 @@ func (m *Mangadex) getChapterInfo(chapterID string) (chapterInfo mangadexChapter } `json:"chapter"` } - if err := fetchJSON(ctx, m.client, imagesEndpoint, &imagesRes); err != nil { + if err := m.client.FetchJSON(ctx, imagesEndpoint, &imagesRes); err != nil { return mangadexChapter{}, err } if strings.ToLower(imagesRes.Result) != "ok" { diff --git a/pkg/sites/manganato_test.go b/pkg/sites/manganato_test.go index 03278dfe..ea1ce090 100644 --- a/pkg/sites/manganato_test.go +++ b/pkg/sites/manganato_test.go @@ -9,6 +9,7 @@ import ( "github.com/Girbons/comics-downloader/internal/logger" "github.com/Girbons/comics-downloader/pkg/config" "github.com/Girbons/comics-downloader/pkg/core" + httpclient "github.com/Girbons/comics-downloader/pkg/http" "github.com/stretchr/testify/require" ) @@ -65,11 +66,17 @@ func TestManganatoScraper(t *testing.T) { server := newManganatoServer() defer server.Close() + client := httpclient.NewComicClient( + httpclient.WithHTTPClient(server.Client()), + httpclient.WithRetry(0, 0), + ) + opts := &config.Options{ URL: server.URL + manganatoListPath, SourceName: "manganato.com", Logger: logger.NewLogger(false, nil), RequestTimeout: config.DefaulltRequestTimeout, + Client: client, } scraper := NewManganato(opts) diff --git a/pkg/sites/mangareader.go b/pkg/sites/mangareader.go index 838f1eb2..16476787 100644 --- a/pkg/sites/mangareader.go +++ b/pkg/sites/mangareader.go @@ -31,7 +31,7 @@ func (m *Mangareader) retrieveImageLinks(comic *core.ComicIssue) ([]string, erro ctx, cancel := m.requestContext() defer cancel() - response, err := fetchHTML(ctx, m.options.Client, comic.Source.URL) + response, err := m.options.Client.FetchHTML(ctx, comic.Source.URL) if err != nil { return nil, err } @@ -62,7 +62,7 @@ func (m *Mangareader) retrieveLastIssue(url string) (string, error) { ctx, cancel := m.requestContext() defer cancel() - response, err := fetchHTML(ctx, m.options.Client, url) + response, err := m.options.Client.FetchHTML(ctx, url) if err != nil { return "", err } @@ -92,7 +92,7 @@ func (m *Mangareader) RetrieveIssueLinks() ([]string, error) { ctx, cancel := m.requestContext() defer cancel() - response, err := fetchHTML(ctx, m.options.Client, url) + response, err := m.options.Client.FetchHTML(ctx, url) if err != nil { return nil, err } diff --git a/pkg/sites/mangareader_test.go b/pkg/sites/mangareader_test.go index ffe46889..95f2a827 100644 --- a/pkg/sites/mangareader_test.go +++ b/pkg/sites/mangareader_test.go @@ -8,6 +8,7 @@ import ( "github.com/Girbons/comics-downloader/internal/logger" "github.com/Girbons/comics-downloader/pkg/config" "github.com/Girbons/comics-downloader/pkg/core" + httpclient "github.com/Girbons/comics-downloader/pkg/http" "github.com/stretchr/testify/require" ) @@ -59,10 +60,16 @@ func TestMangareaderScraper(t *testing.T) { server := newMangareaderServer() defer server.Close() + client := httpclient.NewComicClient( + httpclient.WithHTTPClient(server.Client()), + httpclient.WithRetry(0, 0), + ) + opts := &config.Options{ URL: server.URL + mangareaderIssuePath, Logger: logger.NewLogger(false, nil), RequestTimeout: config.DefaulltRequestTimeout, + Client: client, } scraper := NewMangareader(opts) diff --git a/pkg/sites/mangatown.go b/pkg/sites/mangatown.go index d27edf27..97657681 100644 --- a/pkg/sites/mangatown.go +++ b/pkg/sites/mangatown.go @@ -45,7 +45,7 @@ func (m *Mangatown) retrieveImageLinks(comic *core.ComicIssue) ([]string, error) ctx, cancel := m.requestContext() defer cancel() - response, err := fetchHTML(ctx, m.options.Client, comic.Source.URL) + response, err := m.options.Client.FetchHTML(ctx, comic.Source.URL) if err != nil { return nil, err } @@ -60,7 +60,7 @@ func (m *Mangatown) retrieveImageLinks(comic *core.ComicIssue) ([]string, error) ctx, cancel := m.requestContext() defer cancel() - response, err := fetchHTML(ctx, m.options.Client, link) + response, err := m.options.Client.FetchHTML(ctx, link) if err != nil { return nil, err } @@ -87,7 +87,7 @@ func (m *Mangatown) retrieveLastIssue(url string) (string, error) { ctx, cancel := m.requestContext() defer cancel() - response, err := fetchHTML(ctx, m.options.Client, url) + response, err := m.options.Client.FetchHTML(ctx, url) if err != nil { return "", err } @@ -118,7 +118,7 @@ func (m *Mangatown) RetrieveIssueLinks() ([]string, error) { ctx, cancel := m.requestContext() defer cancel() - response, err := fetchHTML(ctx, m.options.Client, url) + response, err := m.options.Client.FetchHTML(ctx, url) if err != nil { return nil, err } diff --git a/pkg/sites/mangatown_test.go b/pkg/sites/mangatown_test.go index 03abe389..697b18b0 100644 --- a/pkg/sites/mangatown_test.go +++ b/pkg/sites/mangatown_test.go @@ -9,6 +9,7 @@ import ( "github.com/Girbons/comics-downloader/internal/logger" "github.com/Girbons/comics-downloader/pkg/config" "github.com/Girbons/comics-downloader/pkg/core" + httpclient "github.com/Girbons/comics-downloader/pkg/http" "github.com/stretchr/testify/require" ) @@ -72,10 +73,16 @@ func TestMangatownScraper(t *testing.T) { server := newMangatownServer() defer server.Close() + client := httpclient.NewComicClient( + httpclient.WithHTTPClient(server.Client()), + httpclient.WithRetry(0, 0), + ) + opts := &config.Options{ URL: server.URL + mangatownIssuePath, Logger: logger.NewLogger(false, nil), RequestTimeout: config.DefaulltRequestTimeout, + Client: client, } scraper := NewMangatown(opts) diff --git a/pkg/sites/readallcomics.go b/pkg/sites/readallcomics.go index 95a43998..fbd77e7f 100644 --- a/pkg/sites/readallcomics.go +++ b/pkg/sites/readallcomics.go @@ -33,7 +33,7 @@ func (r *Readallcomics) retrieveImageLinks(comic *core.ComicIssue) ([]string, er ctx, cancel := r.requestContext() defer cancel() - response, err := fetchHTML(ctx, r.options.Client, comic.Source.URL) + response, err := r.options.Client.FetchHTML(ctx, comic.Source.URL) if err != nil { return nil, err } @@ -65,7 +65,7 @@ func (r *Readallcomics) getIssues(url string) ([]string, error) { ctx, cancel := r.requestContext() defer cancel() - response, err := fetchHTML(ctx, r.options.Client, url) + response, err := r.options.Client.FetchHTML(ctx, url) if err != nil { return nil, err } diff --git a/pkg/sites/readallcomics_test.go b/pkg/sites/readallcomics_test.go index e276c106..0c533ca1 100644 --- a/pkg/sites/readallcomics_test.go +++ b/pkg/sites/readallcomics_test.go @@ -8,6 +8,7 @@ import ( "github.com/Girbons/comics-downloader/internal/logger" "github.com/Girbons/comics-downloader/pkg/config" "github.com/Girbons/comics-downloader/pkg/core" + httpclient "github.com/Girbons/comics-downloader/pkg/http" "github.com/stretchr/testify/require" ) @@ -55,10 +56,16 @@ func TestReadAllComicsScraper(t *testing.T) { server := newReadAllComicsServer() defer server.Close() + client := httpclient.NewComicClient( + httpclient.WithHTTPClient(server.Client()), + httpclient.WithRetry(0, 0), + ) + opts := &config.Options{ URL: server.URL + readAllIssuePath, Logger: logger.NewLogger(false, nil), RequestTimeout: config.DefaulltRequestTimeout, + Client: client, } scraper := NewReadallcomics(opts) diff --git a/pkg/sites/readcomiconline.go b/pkg/sites/readcomiconline.go index a639bed8..b5d70f34 100644 --- a/pkg/sites/readcomiconline.go +++ b/pkg/sites/readcomiconline.go @@ -79,7 +79,7 @@ func (c *ReadComicOnline) retrieveImageLinks(comic *core.ComicIssue) ([]string, ctx, cancel := c.requestContext() defer cancel() - response, err := fetchHTML(ctx, c.options.Client, fetchURL) + response, err := c.options.Client.FetchHTML(ctx, fetchURL) if err != nil { if c.options.Logger != nil { c.options.Logger.Errorf("readcomiconline: request to %s failed: %v", fetchURL, err) @@ -136,7 +136,7 @@ func (c *ReadComicOnline) retrieveLastIssue(url string) (string, error) { ctx, cancel := c.requestContext() defer cancel() - response, err := fetchHTML(ctx, c.options.Client, url) + response, err := c.options.Client.FetchHTML(ctx, url) if err != nil { return "", err } @@ -173,7 +173,7 @@ func (c *ReadComicOnline) RetrieveIssueLinks() ([]string, error) { ctx, cancel := c.requestContext() defer cancel() - response, err := fetchHTML(ctx, c.options.Client, url) + response, err := c.options.Client.FetchHTML(ctx, url) if err != nil { return nil, err } diff --git a/pkg/sites/readcomiconline_test.go b/pkg/sites/readcomiconline_test.go index a95cdb87..8587a0a4 100644 --- a/pkg/sites/readcomiconline_test.go +++ b/pkg/sites/readcomiconline_test.go @@ -9,6 +9,7 @@ import ( "github.com/Girbons/comics-downloader/internal/logger" "github.com/Girbons/comics-downloader/pkg/config" "github.com/Girbons/comics-downloader/pkg/core" + httpclient "github.com/Girbons/comics-downloader/pkg/http" "github.com/stretchr/testify/require" ) @@ -50,6 +51,11 @@ func TestReadComicOnlineScraper(t *testing.T) { server := newReadComicOnlineServer() defer server.Close() + client := httpclient.NewComicClient( + httpclient.WithHTTPClient(server.Client()), + httpclient.WithRetry(0, 0), + ) + originalBase := baseUrl baseUrl = server.URL defer func() { baseUrl = originalBase }() @@ -58,6 +64,7 @@ func TestReadComicOnlineScraper(t *testing.T) { URL: server.URL + rcoIssuePath, Logger: logger.NewLogger(false, nil), RequestTimeout: config.DefaulltRequestTimeout, + Client: client, } scraper := NewReadComiconline(opts) From ebe90fe0765b4ef7a73bdbd878e1cd7453f22a7e Mon Sep 17 00:00:00 2001 From: ProjectDislocate <173194962+ProjectDislocate@users.noreply.github.com> Date: Tue, 21 Apr 2026 17:24:44 -0400 Subject: [PATCH 53/79] feat: inital switch to site registry --- pkg/sites/comicextra.go | 7 ++++++ pkg/sites/loader.go | 44 ++++++++++++++++++++---------------- pkg/sites/mangadex.go | 7 ++++++ pkg/sites/mangakakalot.go | 7 ++++++ pkg/sites/manganato.go | 7 ++++++ pkg/sites/mangareader.go | 7 ++++++ pkg/sites/mangatown.go | 7 ++++++ pkg/sites/readallcomics.go | 7 ++++++ pkg/sites/readcomiconline.go | 7 ++++++ 9 files changed, 80 insertions(+), 20 deletions(-) diff --git a/pkg/sites/comicextra.go b/pkg/sites/comicextra.go index 5c9f6635..2f4cd09f 100644 --- a/pkg/sites/comicextra.go +++ b/pkg/sites/comicextra.go @@ -14,6 +14,13 @@ import ( "github.com/anaskhan96/soup" ) +func init() { + SupportedSites["comicextra"] = SupportedSite{ + IsEnabled: true, + Loader: func(opts *config.Options) BaseSite { return NewComicextra(opts) }, + } +} + // Comicextra represents comicextra instance. type Comicextra struct { options *config.Options diff --git a/pkg/sites/loader.go b/pkg/sites/loader.go index 5ef38a7d..a87fa5a3 100644 --- a/pkg/sites/loader.go +++ b/pkg/sites/loader.go @@ -13,6 +13,15 @@ import ( "github.com/Girbons/comics-downloader/pkg/util" ) +type SupportedSite struct { + IsEnabled bool + Loader func(*config.Options) BaseSite +} + +// SupportedSites is a map of supported sites and their corresponding loader implementations. +// The key is the site hostname and the value is a struct containing the enabled status and the loader instance. +var SupportedSites = map[string]SupportedSite{} + func initializeCollection(issues []string, options *config.Options, base BaseSite) ([]*core.ComicIssue, error) { var collection []*core.ComicIssue // var err error @@ -156,26 +165,21 @@ func LoadComicFromSource(options *config.Options) ([]*core.ComicIssue, error) { options.Client = httpclient.NewComicClient() } - switch { - case strings.Contains(options.SourceName, "readcomiconline"): - base = NewReadComiconline(options) - case strings.Contains(options.SourceName, "comicextra"): - base = NewComicextra(options) - case strings.Contains(options.SourceName, "mangareader"): - base = NewMangareader(options) - case strings.Contains(options.SourceName, "mangatown"): - base = NewMangatown(options) - case strings.Contains(options.SourceName, "mangadex"): - base = NewMangadex(options) - case strings.Contains(options.SourceName, "readallcomics"): - base = NewReadallcomics(options) - case strings.Contains(options.SourceName, "mangakakalot"): - base = NewMangaKakalot(options) - case strings.Contains(options.SourceName, "manganato"): - base = NewManganato(options) - default: - err = fmt.Errorf("source unknown") - return collection, err + // Look up the site in the registry + var siteFound bool + for siteName, supportedSite := range SupportedSites { + if strings.Contains(options.SourceName, siteName) { + if !supportedSite.IsEnabled { + return collection, fmt.Errorf("source %q is disabled", siteName) + } + base = supportedSite.Loader(options) + siteFound = true + break + } + } + + if !siteFound { + return collection, fmt.Errorf("source unknown") } if options.Logger != nil && options.Debug { diff --git a/pkg/sites/mangadex.go b/pkg/sites/mangadex.go index aa2aebe9..113c4696 100644 --- a/pkg/sites/mangadex.go +++ b/pkg/sites/mangadex.go @@ -49,6 +49,13 @@ func NewMangadex(options *config.Options) *Mangadex { } } +func init() { + SupportedSites["mangadex"] = SupportedSite{ + IsEnabled: true, + Loader: func(opts *config.Options) BaseSite { return NewMangadex(opts) }, + } +} + func (m *Mangadex) requestContext() (context.Context, context.CancelFunc) { return context.WithTimeout(context.Background(), m.options.RequestTimeout) } diff --git a/pkg/sites/mangakakalot.go b/pkg/sites/mangakakalot.go index e4f381f5..35c8f9ed 100644 --- a/pkg/sites/mangakakalot.go +++ b/pkg/sites/mangakakalot.go @@ -16,6 +16,13 @@ func NewMangaKakalot(options *config.Options) *MangaKakalot { } } +func init() { + SupportedSites["mangakakalot"] = SupportedSite{ + IsEnabled: true, + Loader: func(opts *config.Options) BaseSite { return NewMangaKakalot(opts) }, + } +} + // GetInfo extracts the basic info from the given url. func (m *MangaKakalot) GetInfo(url string) (string, string, error) { name, issueNumber, err := MangaKakalotGetInfo(m.options, "mangakakalot.com", url) diff --git a/pkg/sites/manganato.go b/pkg/sites/manganato.go index f72bd61b..419e8701 100644 --- a/pkg/sites/manganato.go +++ b/pkg/sites/manganato.go @@ -16,6 +16,13 @@ func NewManganato(options *config.Options) *Manganato { } } +func init() { + SupportedSites["manganato"] = SupportedSite{ + IsEnabled: true, + Loader: func(opts *config.Options) BaseSite { return NewManganato(opts) }, + } +} + // GetInfo extracts the basic info from the given url. func (m *Manganato) GetInfo(url string) (string, string, error) { name, issueNumber, err := MangaKakalotGetInfo(m.options, "manganato.com", url) diff --git a/pkg/sites/mangareader.go b/pkg/sites/mangareader.go index 16476787..e14412c1 100644 --- a/pkg/sites/mangareader.go +++ b/pkg/sites/mangareader.go @@ -23,6 +23,13 @@ func NewMangareader(options *config.Options) *Mangareader { } } +func init() { + SupportedSites["mangareader"] = SupportedSite{ + IsEnabled: true, + Loader: func(opts *config.Options) BaseSite { return NewMangareader(opts) }, + } +} + func (m *Mangareader) requestContext() (context.Context, context.CancelFunc) { return context.WithTimeout(context.Background(), m.options.RequestTimeout) } diff --git a/pkg/sites/mangatown.go b/pkg/sites/mangatown.go index 97657681..e65c7eda 100644 --- a/pkg/sites/mangatown.go +++ b/pkg/sites/mangatown.go @@ -23,6 +23,13 @@ func NewMangatown(options *config.Options) *Mangatown { } } +func init() { + SupportedSites["mangatown"] = SupportedSite{ + IsEnabled: true, + Loader: func(opts *config.Options) BaseSite { return NewMangatown(opts) }, + } +} + func (m *Mangatown) requestContext() (context.Context, context.CancelFunc) { return context.WithTimeout(context.Background(), m.options.RequestTimeout) } diff --git a/pkg/sites/readallcomics.go b/pkg/sites/readallcomics.go index fbd77e7f..237f4f83 100644 --- a/pkg/sites/readallcomics.go +++ b/pkg/sites/readallcomics.go @@ -25,6 +25,13 @@ func NewReadallcomics(options *config.Options) *Readallcomics { } } +func init() { + SupportedSites["readallcomics"] = SupportedSite{ + IsEnabled: true, + Loader: func(opts *config.Options) BaseSite { return NewReadallcomics(opts) }, + } +} + func (r *Readallcomics) requestContext() (context.Context, context.CancelFunc) { return context.WithTimeout(context.Background(), r.options.RequestTimeout) } diff --git a/pkg/sites/readcomiconline.go b/pkg/sites/readcomiconline.go index b5d70f34..e8baa715 100644 --- a/pkg/sites/readcomiconline.go +++ b/pkg/sites/readcomiconline.go @@ -26,6 +26,13 @@ func NewReadComiconline(options *config.Options) *ReadComicOnline { } } +func init() { + SupportedSites["readcomiconline"] = SupportedSite{ + IsEnabled: true, + Loader: func(opts *config.Options) BaseSite { return NewReadComiconline(opts) }, + } +} + func (c *ReadComicOnline) requestContext() (context.Context, context.CancelFunc) { return context.WithTimeout(context.Background(), c.options.RequestTimeout) } From a550c8a0b6cc64b08f30550abe6df3261ee98bab Mon Sep 17 00:00:00 2001 From: ProjectDislocate <173194962+ProjectDislocate@users.noreply.github.com> Date: Tue, 21 Apr 2026 17:26:00 -0400 Subject: [PATCH 54/79] fix: tests missing proper client init --- pkg/sites/comicextra_test.go | 12 ++++++++++++ pkg/sites/mangakakalot_test.go | 7 +++++++ 2 files changed, 19 insertions(+) diff --git a/pkg/sites/comicextra_test.go b/pkg/sites/comicextra_test.go index fdc396af..fed6395d 100644 --- a/pkg/sites/comicextra_test.go +++ b/pkg/sites/comicextra_test.go @@ -94,11 +94,17 @@ func TestComicExtraRetrieveIssueLinksAll(t *testing.T) { server := newComicExtraServer() defer server.Close() + client := httpclient.NewComicClient( + httpclient.WithHTTPClient(server.Client()), + httpclient.WithRetry(0, 0), + ) + opts := &config.Options{ URL: server.URL + comicExtraListPath, All: true, Logger: logger.NewLogger(false, nil), RequestTimeout: config.DefaulltRequestTimeout, + Client: client, } comicextra := NewComicextra(opts) @@ -114,11 +120,17 @@ func TestComicExtraRetrieveLastIssue(t *testing.T) { server := newComicExtraServer() defer server.Close() + client := httpclient.NewComicClient( + httpclient.WithHTTPClient(server.Client()), + httpclient.WithRetry(0, 0), + ) + opts := &config.Options{ URL: server.URL + comicExtraLastIssuePath, Last: true, Logger: logger.NewLogger(false, nil), RequestTimeout: config.DefaulltRequestTimeout, + Client: client, } comicextra := NewComicextra(opts) diff --git a/pkg/sites/mangakakalot_test.go b/pkg/sites/mangakakalot_test.go index dddb2565..0faefa57 100644 --- a/pkg/sites/mangakakalot_test.go +++ b/pkg/sites/mangakakalot_test.go @@ -10,6 +10,7 @@ import ( "github.com/Girbons/comics-downloader/internal/logger" "github.com/Girbons/comics-downloader/pkg/config" "github.com/Girbons/comics-downloader/pkg/core" + httpclient "github.com/Girbons/comics-downloader/pkg/http" "github.com/stretchr/testify/require" ) @@ -68,11 +69,17 @@ func TestMangaKakalotScraper(t *testing.T) { server := newMangaKakalotServer() defer server.Close() + client := httpclient.NewComicClient( + httpclient.WithHTTPClient(server.Client()), + httpclient.WithRetry(0, 0), + ) + opts := &config.Options{ URL: server.URL + mangaKakalotListPath, SourceName: "mangakakalot.com", Logger: logger.NewLogger(false, nil), RequestTimeout: config.DefaulltRequestTimeout, + Client: client, } scraper := NewMangaKakalot(opts) From 709969b390518adf50743dd77dfc705d5c500b16 Mon Sep 17 00:00:00 2001 From: ProjectDislocate <173194962+ProjectDislocate@users.noreply.github.com> Date: Tue, 21 Apr 2026 17:27:07 -0400 Subject: [PATCH 55/79] feat: detector uses new site registry --- pkg/detector/detector.go | 27 ++----- pkg/detector/detector_test.go | 16 ++++ pkg/sites/loader_test.go | 146 ++++++++++++++++++++++++++++++++++ 3 files changed, 167 insertions(+), 22 deletions(-) diff --git a/pkg/detector/detector.go b/pkg/detector/detector.go index 311c846c..20b91e39 100644 --- a/pkg/detector/detector.go +++ b/pkg/detector/detector.go @@ -3,43 +3,26 @@ package detector import ( "strings" + "github.com/Girbons/comics-downloader/pkg/sites" "github.com/Girbons/comics-downloader/pkg/util" log "github.com/sirupsen/logrus" ) -// SupportedSites are the supported sites. -var SupportedSites = map[string]map[string]bool{ - "comicextra": {"isDisabled": false}, - "mangadex": {"isDisabled": false}, - "mangareader": {"isDisabled": true}, - "mangakakalot": {"isDisabled": false}, - "manganato": {"isDisabled": false}, - "mangatown": {"isDisabled": false}, - "readallcomics": {"isDisabled": false}, - "readcomiconline": {"isDisabled": false}, -} - // DetectSource will look for the url source to check if a source is supported. -func DetectSource(url string) (string, bool, bool) { - var ( - isSupported bool - isDisabled bool - source string - ) - +func DetectSource(url string) (source string, isSupported, isDisabled bool) { source, err := util.URLSource(url) if err != nil { log.Error(err) } - for k, v := range SupportedSites { - if !strings.Contains(source, k) { + for siteName, supportedSite := range sites.SupportedSites { + if !strings.Contains(source, siteName) { continue } isSupported = true - isDisabled = v["isDisabled"] + isDisabled = !supportedSite.IsEnabled } return source, isSupported, isDisabled diff --git a/pkg/detector/detector_test.go b/pkg/detector/detector_test.go index 33ce9983..06da03e7 100644 --- a/pkg/detector/detector_test.go +++ b/pkg/detector/detector_test.go @@ -12,3 +12,19 @@ func TestUnsupportedSource(t *testing.T) { assert.False(t, check) assert.False(t, isDisabled) } + +func TestKnownSupportedSource(t *testing.T) { + source, isSupported, isDisabled := DetectSource("https://comicextra.com/comic/some-comic/issue-1") + + assert.Contains(t, source, "comicextra") + assert.True(t, isSupported) + assert.False(t, isDisabled) +} + +func TestKnownSupportedSourceMangadex(t *testing.T) { + source, isSupported, isDisabled := DetectSource("https://mangadex.org/chapter/abc123") + + assert.Contains(t, source, "mangadex") + assert.True(t, isSupported) + assert.False(t, isDisabled) +} diff --git a/pkg/sites/loader_test.go b/pkg/sites/loader_test.go index ee744bc9..06478db8 100644 --- a/pkg/sites/loader_test.go +++ b/pkg/sites/loader_test.go @@ -167,3 +167,149 @@ func TestExtractIssueNumberForRange(t *testing.T) { }) } } + +func TestLoadComicFromSourceWithRegistry(t *testing.T) { + // Save original registry to restore it after the test + originalRegistry := make(map[string]SupportedSite) + for k, v := range SupportedSites { + originalRegistry[k] = v + } + defer func() { + SupportedSites = originalRegistry + }() + + // Create a test site implementation + testSite := &stubSite{ + issues: []string{"url-1", "url-2"}, + comics: map[string]*core.ComicIssue{ + "url-1": {Name: "test-series", IssueNumber: "1", Source: &core.ComicSource{Name: "test-site", URL: "url-1"}}, + "url-2": {Name: "test-series", IssueNumber: "2", Source: &core.ComicSource{Name: "test-site", URL: "url-2"}}, + }, + } + + // Register the test site in the registry + SupportedSites["test-site"] = SupportedSite{ + IsEnabled: true, + Loader: func(opts *config.Options) BaseSite { + return testSite + }, + } + + options := &config.Options{ + SourceName: "test-site", + URL: "http://test-site.com", + OutputFormat: "pdf", + ImagesFormat: "png", + Logger: logger.NewLogger(false, nil), + } + + collection, err := LoadComicFromSource(options) + require.NoError(t, err) + require.Len(t, collection, 2) + assert.Equal(t, "test-series", collection[0].Name) + assert.Equal(t, "1", collection[0].IssueNumber) + assert.Equal(t, "test-series", collection[1].Name) + assert.Equal(t, "2", collection[1].IssueNumber) +} + +func TestLoadComicFromSourceDisabledSite(t *testing.T) { + // Save original registry + originalRegistry := make(map[string]SupportedSite) + for k, v := range SupportedSites { + originalRegistry[k] = v + } + defer func() { + SupportedSites = originalRegistry + }() + + // Register a disabled test site + testSite := &stubSite{ + issues: []string{"url-1"}, + comics: map[string]*core.ComicIssue{ + "url-1": {Name: "test-series", IssueNumber: "1", Source: &core.ComicSource{Name: "disabled-test-site", URL: "url-1"}}, + }, + } + + SupportedSites["disabled-test-site"] = SupportedSite{ + IsEnabled: false, + Loader: func(opts *config.Options) BaseSite { + return testSite + }, + } + + options := &config.Options{ + SourceName: "disabled-test-site", + URL: "http://disabled-test-site.com", + Logger: logger.NewLogger(false, nil), + } + + collection, err := LoadComicFromSource(options) + require.Error(t, err) + require.Empty(t, collection) + assert.Contains(t, err.Error(), "disabled") +} + +func TestLoadComicFromSourcePartialMatch(t *testing.T) { + // Save original registry + originalRegistry := make(map[string]SupportedSite) + for k, v := range SupportedSites { + originalRegistry[k] = v + } + defer func() { + SupportedSites = originalRegistry + }() + + testSite := &stubSite{ + issues: []string{"url-1"}, + comics: map[string]*core.ComicIssue{ + "url-1": {Name: "my-comic", IssueNumber: "42", Source: &core.ComicSource{Name: "mysite.com", URL: "url-1"}}, + }, + } + + SupportedSites["mysite"] = SupportedSite{ + IsEnabled: true, + Loader: func(opts *config.Options) BaseSite { + return testSite + }, + } + + // Test with full domain name to verify partial matching works + options := &config.Options{ + SourceName: "mysite.com", + URL: "http://mysite.com/comic", + OutputFormat: "pdf", + ImagesFormat: "png", + Logger: logger.NewLogger(false, nil), + } + + collection, err := LoadComicFromSource(options) + require.NoError(t, err) + require.Len(t, collection, 1) + assert.Equal(t, "my-comic", collection[0].Name) + assert.Equal(t, "42", collection[0].IssueNumber) +} + +func TestLoadComicFromSourceUnsupportedSite(t *testing.T) { + // Save original registry + originalRegistry := make(map[string]SupportedSite) + for k, v := range SupportedSites { + originalRegistry[k] = v + } + defer func() { + SupportedSites = originalRegistry + }() + + // Clear registry to ensure no sites are registered + SupportedSites = make(map[string]SupportedSite) + + options := &config.Options{ + SourceName: "unsupported-site.com", + URL: "http://unsupported-site.com/comic", + Logger: logger.NewLogger(false, nil), + } + + collection, err := LoadComicFromSource(options) + require.Error(t, err) + require.Empty(t, collection) + assert.Contains(t, err.Error(), "unknown") +} From 869a42cc75fb1474b9445367e55309e25df7533e Mon Sep 17 00:00:00 2001 From: ProjectDislocate <173194962+ProjectDislocate@users.noreply.github.com> Date: Tue, 21 Apr 2026 17:43:34 -0400 Subject: [PATCH 56/79] feat: ensure issue num is never to long --- pkg/core/core.go | 2 +- pkg/util/path.go | 12 ++++++------ pkg/util/path_test.go | 6 +++--- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/pkg/core/core.go b/pkg/core/core.go index 4ce0a9c8..cb91dead 100644 --- a/pkg/core/core.go +++ b/pkg/core/core.go @@ -257,7 +257,7 @@ func (comic *ComicIssue) DownloadImages(options *config.Options) (*DownloadResul var progress *progressbar.ProgressBar if !options.Debug { - progress = progressbar.NewOptions(len(comic.ImageLinks), progressbar.OptionSetRenderBlankState(true), progressbar.OptionSetDescription(fmt.Sprintf("#%s", comic.IssueNumber))) + progress = progressbar.NewOptions(len(comic.ImageLinks), progressbar.OptionSetRenderBlankState(true), progressbar.OptionSetDescription(fmt.Sprintf("#%s", util.TrimNameLength(comic.IssueNumber)))) } format := util.ImageType(comic.ImagesFormat) diff --git a/pkg/util/path.go b/pkg/util/path.go index 6f32029a..286af2a5 100644 --- a/pkg/util/path.go +++ b/pkg/util/path.go @@ -24,7 +24,7 @@ func createPath(path string) (string, error) { } // TrimNameLength trims the name to a maximum length defined by NameLength constant -func trimNameLength(name string) string { +func TrimNameLength(name string) string { if len(name) > NameLength { return name[:NameLength] } @@ -35,7 +35,7 @@ func trimNameLength(name string) string { // when `createDefaultPath` is false the comic is stored without prepending // the default folder path `comics/source/name/[comic.format]`. func PathSetup(createDefaultPath bool, outputFolder, source, name string) (string, error) { - path := fmt.Sprintf("%s/comics/%s/%s/", outputFolder, source, trimNameLength(name)) + path := fmt.Sprintf("%s/comics/%s/%s/", outputFolder, source, TrimNameLength(name)) if !createDefaultPath { path = fmt.Sprintf("%s/", outputFolder) @@ -48,10 +48,10 @@ func PathSetup(createDefaultPath bool, outputFolder, source, name string) (strin // when `createDefaultPath` is false the images are stored without prepending // the default folder path `comics/source/name/[comic.format]`. func ImagesPathSetup(createDefaultPath bool, outputFolder, source, name, issueFolderName, issueNumber string) (string, error) { - path := fmt.Sprintf("%s/comics/%s/%s/images-%s/", outputFolder, source, trimNameLength(name), trimNameLength(issueNumber)) + path := fmt.Sprintf("%s/comics/%s/%s/images-%s/", outputFolder, source, TrimNameLength(name), TrimNameLength(issueNumber)) if !createDefaultPath { - path = fmt.Sprintf("%s/%s", outputFolder, trimNameLength(issueFolderName+issueNumber)) + path = fmt.Sprintf("%s/%s", outputFolder, TrimNameLength(issueFolderName+issueNumber)) } return createPath(path) @@ -76,7 +76,7 @@ func DirectoryOrFileDoesNotExist(filePath string) bool { // GetPathToFile returns the path where the file should be saved. func GetPathToFile(dir, name, issueNumber, format string, issueNumberOnly bool) string { if issueNumberOnly { - return fmt.Sprintf("%s/%s.%s", dir, trimNameLength(issueNumber), format) + return fmt.Sprintf("%s/%s.%s", dir, TrimNameLength(issueNumber), format) } - return fmt.Sprintf("%s/%s-%s.%s", dir, trimNameLength(name), trimNameLength(issueNumber), format) + return fmt.Sprintf("%s/%s-%s.%s", dir, TrimNameLength(name), TrimNameLength(issueNumber), format) } diff --git a/pkg/util/path_test.go b/pkg/util/path_test.go index 33ef8cee..e2cdafe8 100644 --- a/pkg/util/path_test.go +++ b/pkg/util/path_test.go @@ -40,9 +40,9 @@ func TestTrimNameLength(t *testing.T) { exactLengthName := strings.Repeat("a", NameLength) longName := strings.Repeat("b", NameLength+10) - assert.Equal(t, shortName, trimNameLength(shortName)) - assert.Equal(t, exactLengthName, trimNameLength(exactLengthName)) - assert.Equal(t, strings.Repeat("b", NameLength), trimNameLength(longName)) + assert.Equal(t, shortName, TrimNameLength(shortName)) + assert.Equal(t, exactLengthName, TrimNameLength(exactLengthName)) + assert.Equal(t, strings.Repeat("b", NameLength), TrimNameLength(longName)) } func TestPathSetupTrimsComicName(t *testing.T) { From d079620152b543909cfff07805af019de100b167 Mon Sep 17 00:00:00 2001 From: ProjectDislocate <173194962+ProjectDislocate@users.noreply.github.com> Date: Tue, 21 Apr 2026 17:45:26 -0400 Subject: [PATCH 57/79] fix: undo last commit --- pkg/core/core.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/core/core.go b/pkg/core/core.go index cb91dead..28834c1a 100644 --- a/pkg/core/core.go +++ b/pkg/core/core.go @@ -185,6 +185,7 @@ func (comic *ComicIssue) makeCBRZ(options *config.Options, images *DownloadResul } zipArchiveName := filepath.Join(dir, fmt.Sprintf("%s.zip", comic.IssueNumber)) + // TODO: check if path exists and if the file already exists, to avoid overwriting existing files or creating duplicate files when the same issue is downloaded multiple times newName := util.GetPathToFile(dir, comic.Name, comic.IssueNumber, comic.OutputFormat.String(), options.IssueNumberNameOnly) out, err := os.Create(zipArchiveName) @@ -257,7 +258,7 @@ func (comic *ComicIssue) DownloadImages(options *config.Options) (*DownloadResul var progress *progressbar.ProgressBar if !options.Debug { - progress = progressbar.NewOptions(len(comic.ImageLinks), progressbar.OptionSetRenderBlankState(true), progressbar.OptionSetDescription(fmt.Sprintf("#%s", util.TrimNameLength(comic.IssueNumber)))) + progress = progressbar.NewOptions(len(comic.ImageLinks), progressbar.OptionSetRenderBlankState(true), progressbar.OptionSetDescription(fmt.Sprintf("#%s", comic.IssueNumber))) } format := util.ImageType(comic.ImagesFormat) From 3ee11f28ae8b4b55645ce15aa45fed7e18eb6ef4 Mon Sep 17 00:00:00 2001 From: ProjectDislocate <173194962+ProjectDislocate@users.noreply.github.com> Date: Tue, 21 Apr 2026 17:47:59 -0400 Subject: [PATCH 58/79] fix: prevent trailing spaces in dir name --- pkg/util/path.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/pkg/util/path.go b/pkg/util/path.go index 286af2a5..f308fa23 100644 --- a/pkg/util/path.go +++ b/pkg/util/path.go @@ -4,6 +4,7 @@ import ( "fmt" "os" "path/filepath" + "strings" ) const NameLength = 100 @@ -35,7 +36,7 @@ func TrimNameLength(name string) string { // when `createDefaultPath` is false the comic is stored without prepending // the default folder path `comics/source/name/[comic.format]`. func PathSetup(createDefaultPath bool, outputFolder, source, name string) (string, error) { - path := fmt.Sprintf("%s/comics/%s/%s/", outputFolder, source, TrimNameLength(name)) + path := fmt.Sprintf("%s/comics/%s/%s/", outputFolder, source, strings.TrimSpace(TrimNameLength(name))) if !createDefaultPath { path = fmt.Sprintf("%s/", outputFolder) @@ -48,10 +49,10 @@ func PathSetup(createDefaultPath bool, outputFolder, source, name string) (strin // when `createDefaultPath` is false the images are stored without prepending // the default folder path `comics/source/name/[comic.format]`. func ImagesPathSetup(createDefaultPath bool, outputFolder, source, name, issueFolderName, issueNumber string) (string, error) { - path := fmt.Sprintf("%s/comics/%s/%s/images-%s/", outputFolder, source, TrimNameLength(name), TrimNameLength(issueNumber)) + path := fmt.Sprintf("%s/comics/%s/%s/images-%s/", outputFolder, source, strings.TrimSpace(TrimNameLength(name)), strings.TrimSpace(TrimNameLength(issueNumber))) if !createDefaultPath { - path = fmt.Sprintf("%s/%s", outputFolder, TrimNameLength(issueFolderName+issueNumber)) + path = fmt.Sprintf("%s/%s", outputFolder, strings.TrimSpace(TrimNameLength(issueFolderName+issueNumber))) } return createPath(path) @@ -76,7 +77,7 @@ func DirectoryOrFileDoesNotExist(filePath string) bool { // GetPathToFile returns the path where the file should be saved. func GetPathToFile(dir, name, issueNumber, format string, issueNumberOnly bool) string { if issueNumberOnly { - return fmt.Sprintf("%s/%s.%s", dir, TrimNameLength(issueNumber), format) + return fmt.Sprintf("%s/%s.%s", dir, strings.TrimSpace(TrimNameLength(issueNumber)), format) } - return fmt.Sprintf("%s/%s-%s.%s", dir, TrimNameLength(name), TrimNameLength(issueNumber), format) + return fmt.Sprintf("%s/%s-%s.%s", dir, strings.TrimSpace(TrimNameLength(name)), strings.TrimSpace(TrimNameLength(issueNumber)), format) } From 0451b07e8f0240f5f3138056111114aa7f5eddbd Mon Sep 17 00:00:00 2001 From: ProjectDislocate <173194962+ProjectDislocate@users.noreply.github.com> Date: Tue, 21 Apr 2026 18:06:03 -0400 Subject: [PATCH 59/79] fix: files overwriting each other --- pkg/core/core.go | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/pkg/core/core.go b/pkg/core/core.go index 28834c1a..7d6ca51f 100644 --- a/pkg/core/core.go +++ b/pkg/core/core.go @@ -184,20 +184,32 @@ func (comic *ComicIssue) makeCBRZ(options *config.Options, images *DownloadResul return err } - zipArchiveName := filepath.Join(dir, fmt.Sprintf("%s.zip", comic.IssueNumber)) - // TODO: check if path exists and if the file already exists, to avoid overwriting existing files or creating duplicate files when the same issue is downloaded multiple times newName := util.GetPathToFile(dir, comic.Name, comic.IssueNumber, comic.OutputFormat.String(), options.IssueNumberNameOnly) + if _, statErr := os.Stat(newName); statErr == nil { + if options.Logger != nil { + options.Logger.Infof("Skipping %s because it already exists: %s", strings.ToUpper(comic.OutputFormat.String()), newName) + } + return nil + } else if !os.IsNotExist(statErr) { + return statErr + } - out, err := os.Create(zipArchiveName) + out, err := os.CreateTemp(dir, fmt.Sprintf("%s-*.zip", comic.IssueNumber)) if err != nil { return err } + zipArchiveName := out.Name() defer func() { if out != nil { if closeErr := out.Close(); closeErr != nil && options.Logger != nil { options.Logger.Errorf("failed to close archive %s: %v", zipArchiveName, closeErr) } } + if zipArchiveName != "" { + if removeErr := os.Remove(zipArchiveName); removeErr != nil && !os.IsNotExist(removeErr) && options.Logger != nil { + options.Logger.Errorf("failed to cleanup temp archive %s: %v", zipArchiveName, removeErr) + } + } }() fileMap := make(map[string]string) From 9163d4ad0ce63c4f3f5a325be00d5a963da65677 Mon Sep 17 00:00:00 2001 From: ProjectDislocate <173194962+ProjectDislocate@users.noreply.github.com> Date: Tue, 21 Apr 2026 18:08:36 -0400 Subject: [PATCH 60/79] feat: add http proxy support --- cmd/app/downloader.go | 28 ++++++++++++++++++++++++++ cmd/app/runner_test.go | 40 +++++++++++++++++++++++++++++++++++++ cmd/downloader/main.go | 3 +++ cmd/downloader/main_test.go | 8 ++++++++ pkg/config/options.go | 1 + 5 files changed, 80 insertions(+) diff --git a/cmd/app/downloader.go b/cmd/app/downloader.go index 63a49d29..b94dfb43 100644 --- a/cmd/app/downloader.go +++ b/cmd/app/downloader.go @@ -4,6 +4,8 @@ import ( "context" "errors" "fmt" + "net/http" + urlpkg "net/url" "os" "strings" "time" @@ -219,9 +221,35 @@ func buildClientOptions(base config.Options) []httpclient.Option { })) } + if proxyClient, ok := proxyHTTPClient(base.HTTPProxy); ok { + opts = append(opts, httpclient.WithHTTPClient(proxyClient)) + } + return opts } +func proxyHTTPClient(rawProxy string) (*http.Client, bool) { + trimmed := strings.TrimSpace(rawProxy) + if trimmed == "" { + return nil, false + } + + proxyURL, err := urlpkg.Parse(trimmed) + if err != nil || proxyURL.Scheme == "" || proxyURL.Host == "" { + return nil, false + } + + baseTransport, ok := http.DefaultTransport.(*http.Transport) + if !ok { + return nil, false + } + + transport := baseTransport.Clone() + transport.Proxy = http.ProxyURL(proxyURL) + + return &http.Client{Transport: transport}, true +} + func mergeUserAgents(defaultAgent string, provided []string) []string { candidates := append([]string{defaultAgent}, provided...) seen := make(map[string]struct{}, len(candidates)) diff --git a/cmd/app/runner_test.go b/cmd/app/runner_test.go index 3d1073eb..a91a4a10 100644 --- a/cmd/app/runner_test.go +++ b/cmd/app/runner_test.go @@ -1,6 +1,8 @@ package app import ( + "net/http" + "net/url" "strings" "testing" @@ -58,3 +60,41 @@ func TestBuildClientOptionsCacheToggle(t *testing.T) { t.Fatalf("expected response cache to be disabled when no-cache is set") } } + +func TestBuildClientOptionsProxy(t *testing.T) { + client := httpclient.NewComicClient(buildClientOptions(config.Options{HTTPProxy: "http://127.0.0.1:8080"})...) + + transport, ok := client.HTTPClient().Transport.(*http.Transport) + if !ok { + t.Fatalf("expected *http.Transport, got %T", client.HTTPClient().Transport) + } + if transport.Proxy == nil { + t.Fatalf("expected proxy function to be configured") + } + + reqURL, err := url.Parse("https://example.com") + if err != nil { + t.Fatalf("failed to parse test URL: %v", err) + } + + proxyURL, err := transport.Proxy(&http.Request{URL: reqURL}) + if err != nil { + t.Fatalf("unexpected proxy resolution error: %v", err) + } + if proxyURL == nil { + t.Fatalf("expected resolved proxy URL") + } + if proxyURL.String() != "http://127.0.0.1:8080" { + t.Fatalf("expected proxy URL %q, got %q", "http://127.0.0.1:8080", proxyURL.String()) + } +} + +func TestProxyHTTPClientRejectsInvalidValue(t *testing.T) { + client, ok := proxyHTTPClient("not-a-url") + if ok { + t.Fatalf("expected invalid proxy to be rejected") + } + if client != nil { + t.Fatalf("expected nil client for invalid proxy") + } +} diff --git a/cmd/downloader/main.go b/cmd/downloader/main.go index 3e951124..dd12e718 100644 --- a/cmd/downloader/main.go +++ b/cmd/downloader/main.go @@ -47,6 +47,7 @@ var ( // request customization userAgentsCSV string sessionCookie string + httpProxy string noCache bool requestTimeout time.Duration // throttling @@ -74,6 +75,7 @@ func init() { flag.StringVar(&issueFolderName, "issue-folder-name", "issue-", "Folder name where each issue/chapter will be saved, default 'issue-#'") flag.StringVar(&userAgentsCSV, "user-agents", "", "Comma-separated list of alternative User-Agent values to rotate per request") flag.StringVar(&sessionCookie, "session-cookie", "", "Custom Cookie header value (e.g., cf_clearance=...; other=...) for protected sources") + flag.StringVar(&httpProxy, "http-proxy", "", "HTTP/HTTPS proxy URL used for outbound requests (e.g., http://127.0.0.1:8080)") flag.BoolVar(&noCache, "no-cache", false, "Disable in-memory metadata request caching") flag.DurationVar(&requestTimeout, "request-timeout", config.DefaulltRequestTimeout, "Timeout for HTTP requests (e.g., 8s)") flag.DurationVar(&requestDelay, "request-delay", config.DefaultRequestDelay, "Base delay inserted before downloading each image (e.g. 500ms)") @@ -103,6 +105,7 @@ func buildOptions() config.Options { IssueFolderName: issueFolderName, UserAgents: splitAndTrim(userAgentsCSV), SessionCookie: strings.TrimSpace(sessionCookie), + HTTPProxy: strings.TrimSpace(httpProxy), NoCache: noCache, RequestTimeout: requestTimeout, RequestDelay: requestDelay, diff --git a/cmd/downloader/main_test.go b/cmd/downloader/main_test.go index 8623eddf..8004e967 100644 --- a/cmd/downloader/main_test.go +++ b/cmd/downloader/main_test.go @@ -25,6 +25,7 @@ func TestBuildOptionsCopiesGlobals(t *testing.T) { issueFolderName string userAgentsCSV string sessionCookie string + httpProxy string noCache bool }{ debug: debug, @@ -46,6 +47,7 @@ func TestBuildOptionsCopiesGlobals(t *testing.T) { issueFolderName: issueFolderName, userAgentsCSV: userAgentsCSV, sessionCookie: sessionCookie, + httpProxy: httpProxy, noCache: noCache, } defer func() { @@ -68,6 +70,7 @@ func TestBuildOptionsCopiesGlobals(t *testing.T) { issueFolderName = prev.issueFolderName userAgentsCSV = prev.userAgentsCSV sessionCookie = prev.sessionCookie + httpProxy = prev.httpProxy noCache = prev.noCache }() @@ -90,6 +93,7 @@ func TestBuildOptionsCopiesGlobals(t *testing.T) { issueFolderName = "chapter-" userAgentsCSV = "UA1, UA2 ," sessionCookie = "cf_clearance=abc123; other=value" + httpProxy = "http://127.0.0.1:8080" noCache = true opts := buildOptions() @@ -126,6 +130,10 @@ func TestBuildOptionsCopiesGlobals(t *testing.T) { t.Fatalf("expected session cookie to be copied, got %q", opts.SessionCookie) } + if opts.HTTPProxy != "http://127.0.0.1:8080" { + t.Fatalf("expected http proxy to be copied, got %q", opts.HTTPProxy) + } + if !opts.NoCache { t.Fatalf("expected no-cache option to be copied, got %+v", opts) } diff --git a/pkg/config/options.go b/pkg/config/options.go index 869a04d7..f46a4dce 100644 --- a/pkg/config/options.go +++ b/pkg/config/options.go @@ -39,6 +39,7 @@ type Options struct { UserAgents []string SessionCookie string + HTTPProxy string NoCache bool RequestDelay time.Duration RequestDelayJitter time.Duration From adc2ec76bc82b10e86a063957a72700516e02232 Mon Sep 17 00:00:00 2001 From: ProjectDislocate <173194962+ProjectDislocate@users.noreply.github.com> Date: Tue, 21 Apr 2026 18:10:43 -0400 Subject: [PATCH 61/79] refactor: make clear what the output format is --- cmd/downloader/main.go | 6 ++--- cmd/downloader/main_test.go | 2 +- cmd/gui/gui.go | 24 ++++++++++---------- pkg/config/options.go | 2 +- pkg/core/core.go | 8 +++---- pkg/core/core_test.go | 44 ++++++++++++++++++------------------- pkg/core/metadata.go | 6 ++--- pkg/sites/loader.go | 4 ++-- pkg/sites/loader_test.go | 32 +++++++++++++-------------- pkg/util/image.go | 10 ++++----- 10 files changed, 69 insertions(+), 69 deletions(-) diff --git a/cmd/downloader/main.go b/cmd/downloader/main.go index dd12e718..0256f57e 100644 --- a/cmd/downloader/main.go +++ b/cmd/downloader/main.go @@ -59,7 +59,7 @@ func init() { flag.BoolVar(&debug, "debug", false, "Shows Debug log") flag.BoolVar(&all, "all", false, "Download all issues of the Comic or Comics") flag.BoolVar(&daemon, "daemon", false, "Run the download as daemon") - flag.BoolVar(&imagesOnly, "images-only", false, "Download comic/manga images") + flag.BoolVar(&imagesOnly, "images-only", false, "Download comic/manga images without creating a PDF/CBZ/EPUB") flag.BoolVar(&last, "last", false, "Download the last Comic issue") flag.BoolVar(&versionFlag, "version", false, "Display release version") flag.BoolVar(&createDefaultPath, "create-default-path", true, "Using this flag your comics/issue will be downloaded without prepending the default folder structure, `comics/[source]/[name]/`") @@ -67,7 +67,7 @@ func init() { flag.BoolVar(&forceAspect, "force-aspect", false, "Force images to A4 Portrait aspect ratio") flag.StringVar(&outputFormat, "format", "pdf", "Comic format output, supported formats are pdf,epub,cbr,cbz") flag.StringVar(&customComicName, "custom-comic-name", "", "Use a custom name for the comic output.") - flag.StringVar(&imagesFormat, "images-format", "jpg", "To use with `images-only` flag, choose the image format, available png,jpeg,img") + flag.StringVar(&imagesFormat, "images-format", "jpg", "Choose the output image format, available png,jpeg,img") flag.BoolVar(&issueNumberNameOnly, "issue-number-only", false, "Force only saving with issue number instead of chapter name + issue number.") flag.StringVar(&url, "url", "", "Comic URL or Comic URLS by separating each site with a comma without the use of spaces") flag.StringVar(&outputFolder, "output", "", "Folder where the comics will be saved") @@ -91,7 +91,7 @@ func buildOptions() config.Options { Last: last, Country: country, ImagesOnly: imagesOnly, - ImagesFormat: imagesFormat, + OutputImagesFormat: imagesFormat, IssueNumberNameOnly: issueNumberNameOnly, URL: url, ForceAspect: forceAspect, diff --git a/cmd/downloader/main_test.go b/cmd/downloader/main_test.go index 8004e967..cb02bfc6 100644 --- a/cmd/downloader/main_test.go +++ b/cmd/downloader/main_test.go @@ -102,7 +102,7 @@ func TestBuildOptionsCopiesGlobals(t *testing.T) { t.Fatalf("expected boolean flags to be copied into options: %+v", opts) } - if opts.ImagesFormat != "png" || opts.Country != "jp" || opts.OutputFormat != "epub" { + if opts.OutputImagesFormat != "png" || opts.Country != "jp" || opts.OutputFormat != "epub" { t.Fatalf("expected string values to be copied, got %+v", opts) } diff --git a/cmd/gui/gui.go b/cmd/gui/gui.go index 64f80372..87c0fa48 100644 --- a/cmd/gui/gui.go +++ b/cmd/gui/gui.go @@ -43,18 +43,18 @@ func (d *Downloader) ClearOutputFolderField() { // Submit calls the downloader api with the given options. func (d *Downloader) Submit() { opts := &config.Options{ - Debug: d.Debug.Checked, - All: d.AllChapters.Checked, - Last: d.LastChapter.Checked, - URL: strings.TrimSpace(d.URL.Text), - OutputFormat: d.Format.Selected, - Country: d.Country.Text, - ImagesFormat: d.ImagesFormat.Selected, - ImagesOnly: d.ImagesOnly.Checked, - OutputFolder: d.OutputFolder.Text, - CreateDefaultPath: d.CreateDefaultPath.Checked, - IssuesRange: d.IssuesRange.Text, - CustomComicName: d.CustomComicName.Text, + Debug: d.Debug.Checked, + All: d.AllChapters.Checked, + Last: d.LastChapter.Checked, + URL: strings.TrimSpace(d.URL.Text), + OutputFormat: d.Format.Selected, + Country: d.Country.Text, + OutputImagesFormat: d.ImagesFormat.Selected, + ImagesOnly: d.ImagesOnly.Checked, + OutputFolder: d.OutputFolder.Text, + CreateDefaultPath: d.CreateDefaultPath.Checked, + IssuesRange: d.IssuesRange.Text, + CustomComicName: d.CustomComicName.Text, } go downloader.GuiRun(opts) diff --git a/pkg/config/options.go b/pkg/config/options.go index f46a4dce..d1cbf366 100644 --- a/pkg/config/options.go +++ b/pkg/config/options.go @@ -24,7 +24,7 @@ type Options struct { ImagesOnly bool Daemon bool DaemonTimeout int - ImagesFormat string + OutputImagesFormat string Country string OutputFormat string CustomComicName string diff --git a/pkg/core/core.go b/pkg/core/core.go index 7d6ca51f..b6456f0b 100644 --- a/pkg/core/core.go +++ b/pkg/core/core.go @@ -119,7 +119,7 @@ func (comic *ComicIssue) makePDF(options *config.Options, images *DownloadResult pdf := gofpdf.New("P", "mm", "A4", "") - imageOptions := gofpdf.ImageOptions{ImageType: util.ImageType(comic.ImagesFormat), ReadDpi: true, AllowNegativePosition: false} + imageOptions := gofpdf.ImageOptions{ImageType: util.ImageType(comic.OutputImagesFormat), ReadDpi: true, AllowNegativePosition: false} for _, fileName := range images.FilePaths { mmWd = 210.0 mmHt = 297.0 @@ -273,7 +273,7 @@ func (comic *ComicIssue) DownloadImages(options *config.Options) (*DownloadResul progress = progressbar.NewOptions(len(comic.ImageLinks), progressbar.OptionSetRenderBlankState(true), progressbar.OptionSetDescription(fmt.Sprintf("#%s", comic.IssueNumber))) } - format := util.ImageType(comic.ImagesFormat) + outputFormat := util.ImageType(comic.OutputImagesFormat) requestDelay := options.RequestDelay requestJitter := options.RequestDelayJitter @@ -403,7 +403,7 @@ func (comic *ComicIssue) DownloadImages(options *config.Options) (*DownloadResul options.Logger.Errorf("Unexpected content type '%s' while downloading image number: %d - url: %s (bytes=%d, snippet_base64=%s)", contentType, job.index, job.link, len(data), snippet) } - fileName := fmt.Sprintf("%04d-image.%s", job.index, format) + fileName := fmt.Sprintf("%04d-image.%s", job.index, outputFormat) targetPath := filepath.Join(dir, fileName) imgFile, err := os.Create(targetPath) if err != nil { @@ -411,7 +411,7 @@ func (comic *ComicIssue) DownloadImages(options *config.Options) (*DownloadResul } reader := bytes.NewReader(data) - if err := util.SaveImage(imgFile, reader, format, isWebp); err != nil { + if err := util.SaveImage(imgFile, reader, outputFormat, isWebp); err != nil { if options.Logger != nil { reportLen := len(data) if reportLen > sniffLimit { diff --git a/pkg/core/core_test.go b/pkg/core/core_test.go index 3ab308f7..c9af04cb 100644 --- a/pkg/core/core_test.go +++ b/pkg/core/core_test.go @@ -68,11 +68,11 @@ func TestDownloadImagesCreatesFiles(t *testing.T) { opts := newTestOptions(t, server) comic := &ComicIssue{ - Name: "foo", - Source: &ComicSource{Name: "test-source", URL: server.URL}, - IssueNumber: "1", - ImagesFormat: "png", - ImageLinks: buildLinks(server, 3), + Name: "foo", + Source: &ComicSource{Name: "test-source", URL: server.URL}, + IssueNumber: "1", + OutputImagesFormat: "png", + ImageLinks: buildLinks(server, 3), } result, err := comic.DownloadImages(opts) @@ -96,12 +96,12 @@ func TestMakeComicPDF(t *testing.T) { opts := newTestOptions(t, server) comic := &ComicIssue{ - Name: "foo", - Source: &ComicSource{Name: "test-source", URL: server.URL}, - IssueNumber: "1", - OutputFormat: PDF, - ImagesFormat: "png", - ImageLinks: buildLinks(server, 2), + Name: "foo", + Source: &ComicSource{Name: "test-source", URL: server.URL}, + IssueNumber: "1", + OutputFormat: PDF, + OutputImagesFormat: "png", + ImageLinks: buildLinks(server, 2), } require.NoError(t, comic.MakeComic(opts)) @@ -117,13 +117,13 @@ func TestMakeComicEPUB(t *testing.T) { opts := newTestOptions(t, server) comic := &ComicIssue{ - Name: "bar", - Source: &ComicSource{Name: "test-source", URL: server.URL}, - IssueNumber: "42", - OutputFormat: EPUB, - ImagesFormat: "png", - ImageLinks: buildLinks(server, 2), - SeriesMetadata: &SeriesMetadata{}, + Name: "bar", + Source: &ComicSource{Name: "test-source", URL: server.URL}, + IssueNumber: "42", + OutputFormat: EPUB, + OutputImagesFormat: "png", + ImageLinks: buildLinks(server, 2), + SeriesMetadata: &SeriesMetadata{}, } require.NoError(t, comic.MakeComic(opts)) @@ -141,10 +141,10 @@ func TestMakeComicCBZ(t *testing.T) { comic := &ComicIssue{ Name: "baz", - IssueNumber: "7", - OutputFormat: CBZ, - ImagesFormat: "png", - ImageLinks: buildLinks(server, 2), + IssueNumber: "7", + OutputFormat: CBZ, + OutputImagesFormat: "png", + ImageLinks: buildLinks(server, 2), Source: &ComicSource{Name: "test-source", URL: server.URL}, SeriesMetadata: &SeriesMetadata{ diff --git a/pkg/core/metadata.go b/pkg/core/metadata.go index 7885b4d2..079f13f4 100644 --- a/pkg/core/metadata.go +++ b/pkg/core/metadata.go @@ -91,9 +91,9 @@ type ComicIssue struct { ReleaseDate *time.Time ComicFormat *ComicFormat - ImageLinks []string - OutputFormat ComicOutputFormat - ImagesFormat string + ImageLinks []string + OutputFormat ComicOutputFormat + OutputImagesFormat string // the image format to use when saving images Source *ComicSource SeriesMetadata *SeriesMetadata diff --git a/pkg/sites/loader.go b/pkg/sites/loader.go index a87fa5a3..ea30b99d 100644 --- a/pkg/sites/loader.go +++ b/pkg/sites/loader.go @@ -78,8 +78,8 @@ func initializeCollection(issues []string, options *config.Options, base BaseSit Name: name, IssueNumber: issueNumber, - OutputFormat: outputFormat, - ImagesFormat: options.ImagesFormat, + OutputFormat: outputFormat, + OutputImagesFormat: options.OutputImagesFormat, Source: &core.ComicSource{ Name: options.SourceName, diff --git a/pkg/sites/loader_test.go b/pkg/sites/loader_test.go index 06478db8..782b1bd0 100644 --- a/pkg/sites/loader_test.go +++ b/pkg/sites/loader_test.go @@ -37,12 +37,12 @@ func (s *stubSite) RetrieveIssueLinks() ([]string, error) { func TestInitializeCollectionFiltersIssues(t *testing.T) { options := &config.Options{ - SourceName: "test-source", - OutputFormat: "pdf", - ImagesFormat: "png", - IssuesRange: "1-2", - All: true, - Logger: logger.NewLogger(false, nil), + SourceName: "test-source", + OutputFormat: "pdf", + OutputImagesFormat: "png", + IssuesRange: "1-2", + All: true, + Logger: logger.NewLogger(false, nil), } site := &stubSite{ @@ -196,11 +196,11 @@ func TestLoadComicFromSourceWithRegistry(t *testing.T) { } options := &config.Options{ - SourceName: "test-site", - URL: "http://test-site.com", - OutputFormat: "pdf", - ImagesFormat: "png", - Logger: logger.NewLogger(false, nil), + SourceName: "test-site", + URL: "http://test-site.com", + OutputFormat: "pdf", + OutputImagesFormat: "png", + Logger: logger.NewLogger(false, nil), } collection, err := LoadComicFromSource(options) @@ -275,11 +275,11 @@ func TestLoadComicFromSourcePartialMatch(t *testing.T) { // Test with full domain name to verify partial matching works options := &config.Options{ - SourceName: "mysite.com", - URL: "http://mysite.com/comic", - OutputFormat: "pdf", - ImagesFormat: "png", - Logger: logger.NewLogger(false, nil), + SourceName: "mysite.com", + URL: "http://mysite.com/comic", + OutputFormat: "pdf", + OutputImagesFormat: "png", + Logger: logger.NewLogger(false, nil), } collection, err := LoadComicFromSource(options) diff --git a/pkg/util/image.go b/pkg/util/image.go index cc7b2f1c..66e7d340 100644 --- a/pkg/util/image.go +++ b/pkg/util/image.go @@ -8,7 +8,7 @@ import ( "image/png" "io" "strings" - + "golang.org/x/image/webp" ) @@ -39,10 +39,10 @@ func ImageType(mimeStr string) (tp string) { // SaveImage saves an image from a given format func SaveImage(w io.Writer, content io.Reader, format string, isWebp bool) error { var ( - img image.Image - err error - ) - + img image.Image + err error + ) + if isWebp { img, err = webp.Decode(content) } else { From 7c16f469e2e176e9bedd7e1210d449f2384157c4 Mon Sep 17 00:00:00 2001 From: ProjectDislocate <173194962+ProjectDislocate@users.noreply.github.com> Date: Tue, 21 Apr 2026 18:12:31 -0400 Subject: [PATCH 62/79] feat: add webp output support --- docs/dev.md | 5 ++++- go.mod | 1 + go.sum | 2 ++ pkg/util/image.go | 4 +++- 4 files changed, 10 insertions(+), 2 deletions(-) diff --git a/docs/dev.md b/docs/dev.md index 37157537..1ed90324 100644 --- a/docs/dev.md +++ b/docs/dev.md @@ -2,7 +2,10 @@ ## Go -You must have [go](https://golang.org/doc/install). +You must have: + +- [go](https://golang.org/doc/install) +- GCC or [MinGW](http://tdm-gcc.tdragon.net/download) ## Installing dependencies diff --git a/go.mod b/go.mod index 7fabb940..2da57b1f 100644 --- a/go.mod +++ b/go.mod @@ -24,6 +24,7 @@ require ( github.com/bodgit/plumbing v1.3.0 // indirect github.com/bodgit/sevenzip v1.6.0 // indirect github.com/bodgit/windows v1.0.1 // indirect + github.com/chai2010/webp v1.4.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707 // indirect github.com/fsnotify/fsnotify v1.4.9 // indirect diff --git a/go.sum b/go.sum index 9d31073d..0ceb2766 100644 --- a/go.sum +++ b/go.sum @@ -39,6 +39,8 @@ github.com/bodgit/windows v1.0.1 h1:tF7K6KOluPYygXa3Z2594zxlkbKPAOvqr97etrGNIz4= github.com/bodgit/windows v1.0.1/go.mod h1:a6JLwrB4KrTR5hBpp8FI9/9W9jJfeQ2h4XDXU74ZCdM= github.com/boombuler/barcode v1.0.0/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/chai2010/webp v1.4.0 h1:6DA2pkkRUPnbOHvvsmGI3He1hBKf/bkRlniAiSGuEko= +github.com/chai2010/webp v1.4.0/go.mod h1:0XVwvZWdjjdxpUEIf7b9g9VkHFnInUSYujwqTLEuldU= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= diff --git a/pkg/util/image.go b/pkg/util/image.go index 66e7d340..0af5ec1b 100644 --- a/pkg/util/image.go +++ b/pkg/util/image.go @@ -9,7 +9,7 @@ import ( "io" "strings" - "golang.org/x/image/webp" + "github.com/chai2010/webp" ) // IMAGEREGEX to extract the image html tag @@ -64,6 +64,8 @@ func SaveImage(w io.Writer, content io.Reader, format string, isWebp bool) error case "png": pngEncoder := png.Encoder{CompressionLevel: png.BestCompression} return pngEncoder.Encode(w, img) + case "webp": + return webp.Encode(w, img, &webp.Options{Lossless: true}) default: return errors.New("format not found") } From dfa4172e10a54d8fe3dd57a5cb3e216e799165dc Mon Sep 17 00:00:00 2001 From: ProjectDislocate <173194962+ProjectDislocate@users.noreply.github.com> Date: Tue, 21 Apr 2026 18:13:31 -0400 Subject: [PATCH 63/79] refactor: type image format --- pkg/core/core.go | 2 +- pkg/util/image.go | 41 +++++++++++++++++++++++++++++------------ pkg/util/image_test.go | 12 ++++++------ 3 files changed, 36 insertions(+), 19 deletions(-) diff --git a/pkg/core/core.go b/pkg/core/core.go index b6456f0b..2558b1ea 100644 --- a/pkg/core/core.go +++ b/pkg/core/core.go @@ -119,7 +119,7 @@ func (comic *ComicIssue) makePDF(options *config.Options, images *DownloadResult pdf := gofpdf.New("P", "mm", "A4", "") - imageOptions := gofpdf.ImageOptions{ImageType: util.ImageType(comic.OutputImagesFormat), ReadDpi: true, AllowNegativePosition: false} + imageOptions := gofpdf.ImageOptions{ImageType: util.ImageType(comic.OutputImagesFormat).String(), ReadDpi: true, AllowNegativePosition: false} for _, fileName := range images.FilePaths { mmWd = 210.0 mmHt = 297.0 diff --git a/pkg/util/image.go b/pkg/util/image.go index 0af5ec1b..6c3947c8 100644 --- a/pkg/util/image.go +++ b/pkg/util/image.go @@ -15,34 +15,51 @@ import ( // IMAGEREGEX to extract the image html tag const IMAGEREGEX = `]+src="([^">]+)"` +type ImageFormat string + +func (f ImageFormat) String() string { + return string(f) +} + +const ( + ImgFormatPNG ImageFormat = "png" + ImgFormatJPG ImageFormat = "jpg" + ImgFormatGIF ImageFormat = "gif" + ImgFormatWEBP ImageFormat = "webp" + ImgFormatIMG ImageFormat = "img" + ImgFormatUnknown ImageFormat = "unknown" +) + // ImageType return the image type -func ImageType(mimeStr string) (tp string) { +func ImageType(mimeStr string) (format ImageFormat) { + mimeStr = strings.ToLower(mimeStr) switch mimeStr { case "image/png", "png": - tp = "png" - case "image/jpg", "jpg": - tp = "jpg" - case "image/jpeg", "jpeg": - tp = "jpg" + format = ImgFormatPNG + case "image/jpg", "jpg", "image/jpeg", "jpeg": + format = ImgFormatJPG case "image/gif", "gif": - tp = "gif" + format = ImgFormatGIF case "image/webp", "webp": - tp = "webp" + format = ImgFormatWEBP case "img": - tp = "img" + format = ImgFormatIMG default: - tp = "unknown" + format = ImgFormatUnknown } return } // SaveImage saves an image from a given format -func SaveImage(w io.Writer, content io.Reader, format string, isWebp bool) error { +func SaveImage(w io.Writer, content io.Reader, outputFormat ImageFormat, isWebp bool) error { var ( img image.Image err error ) + // TODO: we can optimize this by only decoding the image if the output format is different from the input format, otherwise we can just copy the content to the writer without decoding and encoding again + // TODO: add avif support + if isWebp { img, err = webp.Decode(content) } else { @@ -53,7 +70,7 @@ func SaveImage(w io.Writer, content io.Reader, format string, isWebp bool) error return err } - switch strings.ToLower(format) { + switch strings.ToLower(outputFormat.String()) { case "img": _, err = io.Copy(w, content) return err diff --git a/pkg/util/image_test.go b/pkg/util/image_test.go index 287501d9..c33946f1 100644 --- a/pkg/util/image_test.go +++ b/pkg/util/image_test.go @@ -7,10 +7,10 @@ import ( ) func TestImageType(t *testing.T) { - assert.Equal(t, ImageType("image/jpg"), "jpg") - assert.Equal(t, ImageType("image/jpeg"), "jpg") - assert.Equal(t, ImageType("image/png"), "png") - assert.Equal(t, ImageType("image/gif"), "gif") - assert.Equal(t, ImageType("image/webp"), "webp") - assert.Equal(t, ImageType("foo"), "unknown") + assert.Equal(t, ImageType("image/jpg"), ImageFormat("jpg")) + assert.Equal(t, ImageType("image/jpeg"), ImageFormat("jpg")) + assert.Equal(t, ImageType("image/png"), ImageFormat("png")) + assert.Equal(t, ImageType("image/gif"), ImageFormat("gif")) + assert.Equal(t, ImageType("image/webp"), ImageFormat("webp")) + assert.Equal(t, ImageType("foo"), ImageFormat("unknown")) } From 772103206b7922574b803b5d140ab795f595f2ca Mon Sep 17 00:00:00 2001 From: ProjectDislocate <173194962+ProjectDislocate@users.noreply.github.com> Date: Tue, 21 Apr 2026 18:14:08 -0400 Subject: [PATCH 64/79] refactor: tell SaveImage the provided image's format --- pkg/core/core.go | 25 ++++++++++++++++++------- pkg/util/image.go | 4 ++-- 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/pkg/core/core.go b/pkg/core/core.go index 2558b1ea..987ba4a5 100644 --- a/pkg/core/core.go +++ b/pkg/core/core.go @@ -394,13 +394,24 @@ func (comic *ComicIssue) DownloadImages(options *config.Options) (*DownloadResul } isWebp := strings.HasSuffix(strings.ToLower(job.link), ".webp") || strings.Contains(contentType, "image/webp") - if options.Logger != nil && contentType != "" && !strings.HasPrefix(contentType, "image/") { - reportLen := len(data) - if reportLen > sniffLimit { - reportLen = sniffLimit + var inputImageFormat util.ImageFormat + if isWebp { + inputImageFormat = util.ImgFormatWEBP + } else { + inputImageFormat = util.ImageType(contentType) + } + if options.Logger != nil { + // if the content type is present but does not indicate an image + if contentType != "" && !strings.HasPrefix(contentType, "image/") { + reportLen := len(data) + if reportLen > sniffLimit { + reportLen = sniffLimit + } + snippet := base64.StdEncoding.EncodeToString(data[:reportLen]) + options.Logger.Errorf("Unexpected content type '%s' while downloading image number: %d - url: %s (bytes=%d, snippet_base64=%s)", contentType, job.index, job.link, len(data), snippet) + } else if inputImageFormat == util.ImgFormatUnknown { + options.Logger.Warningf("Could not determine image format for image number: %d - comic issue: %s, content type: '%s', url: %s", job.index, comic.IssueNumber, contentType, job.link) } - snippet := base64.StdEncoding.EncodeToString(data[:reportLen]) - options.Logger.Errorf("Unexpected content type '%s' while downloading image number: %d - url: %s (bytes=%d, snippet_base64=%s)", contentType, job.index, job.link, len(data), snippet) } fileName := fmt.Sprintf("%04d-image.%s", job.index, outputFormat) @@ -411,7 +422,7 @@ func (comic *ComicIssue) DownloadImages(options *config.Options) (*DownloadResul } reader := bytes.NewReader(data) - if err := util.SaveImage(imgFile, reader, outputFormat, isWebp); err != nil { + if err := util.SaveImage(imgFile, reader, outputFormat, inputImageFormat); err != nil { if options.Logger != nil { reportLen := len(data) if reportLen > sniffLimit { diff --git a/pkg/util/image.go b/pkg/util/image.go index 6c3947c8..3c5f89b9 100644 --- a/pkg/util/image.go +++ b/pkg/util/image.go @@ -51,7 +51,7 @@ func ImageType(mimeStr string) (format ImageFormat) { } // SaveImage saves an image from a given format -func SaveImage(w io.Writer, content io.Reader, outputFormat ImageFormat, isWebp bool) error { +func SaveImage(w io.Writer, content io.Reader, outputFormat ImageFormat, providedImageFormat ImageFormat) error { var ( img image.Image err error @@ -60,7 +60,7 @@ func SaveImage(w io.Writer, content io.Reader, outputFormat ImageFormat, isWebp // TODO: we can optimize this by only decoding the image if the output format is different from the input format, otherwise we can just copy the content to the writer without decoding and encoding again // TODO: add avif support - if isWebp { + if providedImageFormat == ImgFormatWEBP { img, err = webp.Decode(content) } else { img, _, err = image.Decode(content) From 0d5c871b9038c59ca2a1acc2706ba8061e0fcace Mon Sep 17 00:00:00 2001 From: ProjectDislocate <173194962+ProjectDislocate@users.noreply.github.com> Date: Tue, 21 Apr 2026 18:31:09 -0400 Subject: [PATCH 65/79] feat: add libjpeg support fixes issues with certain images not being decoded properly --- Makefile | 25 ++++++++++++---------- docs/dev.md | 29 ++++++++++++++++++++++++++ go.mod | 5 +++-- go.sum | 2 ++ pkg/util/image.go | 38 +++++++++++++++++++++++++++------- pkg/util/jpeg_codec_libjpeg.go | 18 ++++++++++++++++ pkg/util/jpeg_codec_std.go | 17 +++++++++++++++ 7 files changed, 114 insertions(+), 20 deletions(-) create mode 100644 pkg/util/jpeg_codec_libjpeg.go create mode 100644 pkg/util/jpeg_codec_std.go diff --git a/Makefile b/Makefile index 38a37e3b..b2debb5d 100644 --- a/Makefile +++ b/Makefile @@ -1,39 +1,42 @@ +BUILD_TAGS ?= +GO_BUILD_TAG_FLAGS := $(if $(strip $(BUILD_TAGS)),-tags "$(BUILD_TAGS)",) + help: # this command # [generating help from tasks header] @egrep '^[A-Za-z0-9_-]+:' Makefile osx-build-arm: # Creates Mac OSX - @GOOS=darwin go build -o build/comics-downloader-osx-arm ./cmd/downloader + @GOOS=darwin go build $(GO_BUILD_TAG_FLAGS) -o build/comics-downloader-osx-arm ./cmd/downloader osx-build-x86-64: # Creates Mac OSX - @GOOS=darwin GOARCH=amd64 go build -o build/comics-downloader-osx-x86-64 ./cmd/downloader + @GOOS=darwin GOARCH=amd64 go build $(GO_BUILD_TAG_FLAGS) -o build/comics-downloader-osx-x86-64 ./cmd/downloader windows-x86-64-build: # Creates Windows - @GOOS=windows GOARCH=amd64 go build -o build/comics-downloader-win-x86-64.exe ./cmd/downloader + @GOOS=windows GOARCH=amd64 go build $(GO_BUILD_TAG_FLAGS) -o build/comics-downloader-win-x86-64.exe ./cmd/downloader windows-386-build: # Creates Windows - @GOOS=windows GOARCH=386 go build -o build/comics-downloader-win-386.exe ./cmd/downloader + @GOOS=windows GOARCH=386 go build $(GO_BUILD_TAG_FLAGS) -o build/comics-downloader-win-386.exe ./cmd/downloader linux-x86-64-build: # Creates Linux - @GOOS=linux GOARCH=amd64 go build -o build/comics-downloader-linux-x86-64 ./cmd/downloader + @GOOS=linux GOARCH=amd64 go build $(GO_BUILD_TAG_FLAGS) -o build/comics-downloader-linux-x86-64 ./cmd/downloader linux-386-build: - @GOOS=linux GOARCH=386 go build -o build/comics-downloader-linux-386 ./cmd/downloader + @GOOS=linux GOARCH=386 go build $(GO_BUILD_TAG_FLAGS) -o build/comics-downloader-linux-386 ./cmd/downloader linux-arm-build: # Creates Linux ARM - @GOOS=linux GOARCH=arm go build -o build/comics-downloader-linux-arm ./cmd/downloader + @GOOS=linux GOARCH=arm go build $(GO_BUILD_TAG_FLAGS) -o build/comics-downloader-linux-arm ./cmd/downloader linux-arm64-build: # Creates Linux ARM64 - @GOOS=linux GOARCH=arm64 go build -o build/comics-downloader-linux-arm64 ./cmd/downloader + @GOOS=linux GOARCH=arm64 go build $(GO_BUILD_TAG_FLAGS) -o build/comics-downloader-linux-arm64 ./cmd/downloader osx-gui-build: # Creates osx GUI - @GOOS=darwin go build -o build/comics-downloader-gui-osx ./cmd/gui + @GOOS=darwin go build $(GO_BUILD_TAG_FLAGS) -o build/comics-downloader-gui-osx ./cmd/gui windows-gui-build: # Creates Window GUI executable - @fyne-cross windows -output comics-downloader-gui-windows.exe ./cmd/gui + @fyne-cross windows -output comics-downloader-gui-windows.exe $(GO_BUILD_TAG_FLAGS) ./cmd/gui linux-gui-build: # Creates Linux Gui executable - @fyne-cross linux -output comics-downloader-gui ./cmd/gui + @fyne-cross linux -output comics-downloader-gui $(GO_BUILD_TAG_FLAGS) ./cmd/gui builds: # Creates executables for OSX/Windows/Linux @make linux-386-build diff --git a/docs/dev.md b/docs/dev.md index 1ed90324..51f3e0aa 100644 --- a/docs/dev.md +++ b/docs/dev.md @@ -32,6 +32,35 @@ go build -o comics-downloader-gui ./cmd/gui if you don't want to install extra dependencies to build the GUI version you could use [fyne-cross](https://github.com/lucor/fyne-cross) which requires [Docker](https://www.docker.com/get-started). +### Optional: libjpeg backend for JPEG encode/decode + +Comics Downloader supports an optional JPEG backend using [go-libjpeg](https://github.com/pixiv/go-libjpeg). +The default behavior is to use the Go standard library unless you pass the `libjpeg` build tag. + +Additional requirements for `libjpeg` builds: + +- libjpeg development headers and libraries (for example `libjpeg-dev` or `libjpeg-turbo-devel`) + +Examples: + +```bash +# CLI +go build -tags libjpeg -o comics-downloader ./cmd/downloader + +# GUI +go build -tags libjpeg -o comics-downloader-gui ./cmd/gui +``` + +Using Makefile targets with build tags: + +```bash +# Build all release artifacts using libjpeg backend +make builds BUILD_TAGS=libjpeg + +# Build a specific target with libjpeg backend +make linux-x86-64-build BUILD_TAGS=libjpeg +``` + ## Run Tests ``` diff --git a/go.mod b/go.mod index 2da57b1f..b1e4eb19 100644 --- a/go.mod +++ b/go.mod @@ -7,13 +7,14 @@ require ( github.com/anaskhan96/soup v1.2.5 github.com/beevik/etree v1.6.0 github.com/bmaupin/go-epub v1.1.0 + github.com/chai2010/webp v1.4.0 github.com/dlclark/regexp2 v1.10.0 github.com/jung-kurt/gofpdf v1.16.2 github.com/mholt/archives v0.1.2 + github.com/pixiv/go-libjpeg v0.0.0-20190822045933-3da21a74767d github.com/schollz/progressbar/v2 v2.15.0 github.com/sirupsen/logrus v1.9.3 github.com/stretchr/testify v1.11.1 - golang.org/x/image v0.39.0 golang.org/x/mod v0.35.0 golang.org/x/sync v0.20.0 ) @@ -24,7 +25,6 @@ require ( github.com/bodgit/plumbing v1.3.0 // indirect github.com/bodgit/sevenzip v1.6.0 // indirect github.com/bodgit/windows v1.0.1 // indirect - github.com/chai2010/webp v1.4.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707 // indirect github.com/fsnotify/fsnotify v1.4.9 // indirect @@ -53,6 +53,7 @@ require ( github.com/ulikunitz/xz v0.5.12 // indirect github.com/vincent-petithory/dataurl v0.0.0-20191104211930-d1553a71de50 // indirect go4.org v0.0.0-20230225012048-214862532bf5 // indirect + golang.org/x/image v0.39.0 // indirect golang.org/x/net v0.47.0 // indirect golang.org/x/sys v0.38.0 // indirect golang.org/x/text v0.36.0 // indirect diff --git a/go.sum b/go.sum index 0ceb2766..0d3dcb72 100644 --- a/go.sum +++ b/go.sum @@ -144,6 +144,8 @@ github.com/nwaples/rardecode/v2 v2.1.0/go.mod h1:7uz379lSxPe6j9nvzxUZ+n7mnJNgjsR github.com/phpdave11/gofpdi v1.0.7/go.mod h1:vBmVV0Do6hSBHC8uKUQ71JGW+ZGQq74llk/7bXwjDoI= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pixiv/go-libjpeg v0.0.0-20190822045933-3da21a74767d h1:ls+7AYarUlUSetfnN/DKVNcK6W8mQWc6VblmOm4XwX0= +github.com/pixiv/go-libjpeg v0.0.0-20190822045933-3da21a74767d/go.mod h1:DO7ixpslN6XfbWzeNH9vkS5CF2FQUX81B85rYe9zDxU= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= diff --git a/pkg/util/image.go b/pkg/util/image.go index 3c5f89b9..b5bcc568 100644 --- a/pkg/util/image.go +++ b/pkg/util/image.go @@ -1,10 +1,10 @@ package util import ( + "bufio" "errors" "image" "image/gif" - "image/jpeg" "image/png" "io" "strings" @@ -57,14 +57,15 @@ func SaveImage(w io.Writer, content io.Reader, outputFormat ImageFormat, provide err error ) + if strings.EqualFold(outputFormat.String(), ImgFormatIMG.String()) { + _, err = io.Copy(w, content) + return err + } + // TODO: we can optimize this by only decoding the image if the output format is different from the input format, otherwise we can just copy the content to the writer without decoding and encoding again // TODO: add avif support - if providedImageFormat == ImgFormatWEBP { - img, err = webp.Decode(content) - } else { - img, _, err = image.Decode(content) - } + img, err = decodeInputImage(content, providedImageFormat) if err != nil { return err @@ -77,7 +78,7 @@ func SaveImage(w io.Writer, content io.Reader, outputFormat ImageFormat, provide case "gif": return gif.Encode(w, img, nil) case "jpg", "jpeg": - return jpeg.Encode(w, img, &jpeg.Options{Quality: 100}) + return encodeJPEG(w, img) case "png": pngEncoder := png.Encoder{CompressionLevel: png.BestCompression} return pngEncoder.Encode(w, img) @@ -87,3 +88,26 @@ func SaveImage(w io.Writer, content io.Reader, outputFormat ImageFormat, provide return errors.New("format not found") } } + +func decodeInputImage(content io.Reader, providedImageFormat ImageFormat) (image.Image, error) { + if providedImageFormat == ImgFormatWEBP { + return webp.Decode(content) + } + + bufferedContent := bufio.NewReader(content) + if isJPEGStream(bufferedContent) { + return decodeJPEG(bufferedContent) + } + + img, _, err := image.Decode(bufferedContent) + return img, err +} + +func isJPEGStream(content *bufio.Reader) bool { + header, err := content.Peek(3) + if err != nil { + return false + } + + return header[0] == 0xFF && header[1] == 0xD8 && header[2] == 0xFF +} diff --git a/pkg/util/jpeg_codec_libjpeg.go b/pkg/util/jpeg_codec_libjpeg.go new file mode 100644 index 00000000..72132cc6 --- /dev/null +++ b/pkg/util/jpeg_codec_libjpeg.go @@ -0,0 +1,18 @@ +//go:build libjpeg + +package util + +import ( + "image" + "io" + + libjpeg "github.com/pixiv/go-libjpeg/jpeg" +) + +func decodeJPEG(content io.Reader) (image.Image, error) { + return libjpeg.Decode(content, &libjpeg.DecoderOptions{}) +} + +func encodeJPEG(w io.Writer, img image.Image) error { + return libjpeg.Encode(w, img, &libjpeg.EncoderOptions{Quality: 100}) +} diff --git a/pkg/util/jpeg_codec_std.go b/pkg/util/jpeg_codec_std.go new file mode 100644 index 00000000..11008be1 --- /dev/null +++ b/pkg/util/jpeg_codec_std.go @@ -0,0 +1,17 @@ +//go:build !libjpeg + +package util + +import ( + "image" + "image/jpeg" + "io" +) + +func decodeJPEG(content io.Reader) (image.Image, error) { + return jpeg.Decode(content) +} + +func encodeJPEG(w io.Writer, img image.Image) error { + return jpeg.Encode(w, img, &jpeg.Options{Quality: 100}) +} From 55cf4c99264297d06ca6a6786a2ce5e8fc144daa Mon Sep 17 00:00:00 2001 From: ProjectDislocate <173194962+ProjectDislocate@users.noreply.github.com> Date: Tue, 21 Apr 2026 18:32:07 -0400 Subject: [PATCH 66/79] fix: mangadex redownloading the same comics --- pkg/sites/mangadex.go | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/pkg/sites/mangadex.go b/pkg/sites/mangadex.go index 113c4696..b9bb4ff1 100644 --- a/pkg/sites/mangadex.go +++ b/pkg/sites/mangadex.go @@ -591,22 +591,11 @@ func (m *Mangadex) GetInfo(urlValue string) (string, string, error) { return "", "", err } - var chapterTitle string - if chapter.Volume != nil { - volume := *chapter.Volume - chapterTitle = fmt.Sprintf("Vol %s Chapter %s", volume, chapter.ChapterNumber) - } else { - chapterTitle = fmt.Sprintf("Chapter %s", chapter.ChapterNumber) - } - - if chapter.ChapterTitle != "" { - chapterTitle += fmt.Sprintf(", %s", chapter.ChapterTitle) - } manga, err := m.getMangaInfo(chapter.MangaID) if err != nil { return "", "", err } - return manga.Title, chapterTitle, nil + return manga.Title, chapter.ChapterNumber, nil case "title": manga, err := m.getMangaInfo(parts[4]) From b94ddf5a0a1451fcf392e494abf4a496cabd592d Mon Sep 17 00:00:00 2001 From: ProjectDislocate <173194962+ProjectDislocate@users.noreply.github.com> Date: Tue, 21 Apr 2026 18:33:24 -0400 Subject: [PATCH 67/79] fix: prevent tripple periods causes issues on windows file systems --- pkg/util/util.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/util/util.go b/pkg/util/util.go index 8787e416..a6e982bc 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -62,6 +62,7 @@ func Parse(s string) string { ";", "", "!", "", "?", "", + "...", "", ) return strings.Trim(replacer.Replace(s), " ") From 2225706c904af0deff295f627420c2bd0cd39348 Mon Sep 17 00:00:00 2001 From: ProjectDislocate <173194962+ProjectDislocate@users.noreply.github.com> Date: Tue, 21 Apr 2026 18:35:02 -0400 Subject: [PATCH 68/79] fix: use std lib to encode jpegs go-libjpeg doesn't support encoding as many formats as std go lib --- pkg/util/jpeg_codec_libjpeg.go | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/pkg/util/jpeg_codec_libjpeg.go b/pkg/util/jpeg_codec_libjpeg.go index 72132cc6..55f410ab 100644 --- a/pkg/util/jpeg_codec_libjpeg.go +++ b/pkg/util/jpeg_codec_libjpeg.go @@ -4,15 +4,30 @@ package util import ( "image" + stdjpeg "image/jpeg" "io" libjpeg "github.com/pixiv/go-libjpeg/jpeg" ) +// // https://github.com/pixiv/go-libjpeg/blob/3da21a74767d9ffe29fcad7484ddd745f99e9f4c/jpeg/compress.go#L243 +// var libjpegUnsupportedFormat = errors.New("unsupported image type") + func decodeJPEG(content io.Reader) (image.Image, error) { return libjpeg.Decode(content, &libjpeg.DecoderOptions{}) } func encodeJPEG(w io.Writer, img image.Image) error { - return libjpeg.Encode(w, img, &libjpeg.EncoderOptions{Quality: 100}) + // err := libjpeg.Encode(w, img, &libjpeg.EncoderOptions{Quality: 100}) + // if err == nil { + // return nil + // } + + // if errors.Is(err, libjpegUnsupportedFormat) { + // // try to use stdjpeg to encode the image because libjpeg doesn't support some image type + // return stdjpeg.Encode(w, img, &stdjpeg.Options{Quality: 100}) + // } + // return err + + return stdjpeg.Encode(w, img, &stdjpeg.Options{Quality: 100}) } From d5db676f7bf8ef9af9a0811319a5ddfa69e58fd2 Mon Sep 17 00:00:00 2001 From: ProjectDislocate <173194962+ProjectDislocate@users.noreply.github.com> Date: Tue, 21 Apr 2026 18:36:37 -0400 Subject: [PATCH 69/79] fix: dont recompute image format --- pkg/core/core.go | 2 +- pkg/util/image.go | 18 ++++-------------- 2 files changed, 5 insertions(+), 15 deletions(-) diff --git a/pkg/core/core.go b/pkg/core/core.go index 987ba4a5..a4075a24 100644 --- a/pkg/core/core.go +++ b/pkg/core/core.go @@ -422,7 +422,7 @@ func (comic *ComicIssue) DownloadImages(options *config.Options) (*DownloadResul } reader := bytes.NewReader(data) - if err := util.SaveImage(imgFile, reader, outputFormat, inputImageFormat); err != nil { + if err := util.SaveImage(options.Logger, imgFile, reader, outputFormat, inputImageFormat); err != nil { if options.Logger != nil { reportLen := len(data) if reportLen > sniffLimit { diff --git a/pkg/util/image.go b/pkg/util/image.go index b5bcc568..97fb9bf0 100644 --- a/pkg/util/image.go +++ b/pkg/util/image.go @@ -9,6 +9,7 @@ import ( "io" "strings" + "github.com/Girbons/comics-downloader/internal/logger" "github.com/chai2010/webp" ) @@ -32,8 +33,7 @@ const ( // ImageType return the image type func ImageType(mimeStr string) (format ImageFormat) { - mimeStr = strings.ToLower(mimeStr) - switch mimeStr { + switch strings.ToLower(strings.TrimSpace(mimeStr)) { case "image/png", "png": format = ImgFormatPNG case "image/jpg", "jpg", "image/jpeg", "jpeg": @@ -51,7 +51,7 @@ func ImageType(mimeStr string) (format ImageFormat) { } // SaveImage saves an image from a given format -func SaveImage(w io.Writer, content io.Reader, outputFormat ImageFormat, providedImageFormat ImageFormat) error { +func SaveImage(logger *logger.Logger, w io.Writer, content io.Reader, outputFormat ImageFormat, providedImageFormat ImageFormat) error { var ( img image.Image err error @@ -66,7 +66,6 @@ func SaveImage(w io.Writer, content io.Reader, outputFormat ImageFormat, provide // TODO: add avif support img, err = decodeInputImage(content, providedImageFormat) - if err != nil { return err } @@ -95,19 +94,10 @@ func decodeInputImage(content io.Reader, providedImageFormat ImageFormat) (image } bufferedContent := bufio.NewReader(content) - if isJPEGStream(bufferedContent) { + if providedImageFormat == ImgFormatJPG { return decodeJPEG(bufferedContent) } img, _, err := image.Decode(bufferedContent) return img, err } - -func isJPEGStream(content *bufio.Reader) bool { - header, err := content.Peek(3) - if err != nil { - return false - } - - return header[0] == 0xFF && header[1] == 0xD8 && header[2] == 0xFF -} From bdc7ba73991495f9f713ed43a3d7dad9f434c9e9 Mon Sep 17 00:00:00 2001 From: ProjectDislocate <173194962+ProjectDislocate@users.noreply.github.com> Date: Tue, 21 Apr 2026 18:37:14 -0400 Subject: [PATCH 70/79] feat: add chapter volume to filename --- pkg/core/core.go | 14 +++++++++++--- pkg/core/core_test.go | 6 +++--- pkg/sites/base.go | 1 + pkg/sites/mangadex_test.go | 6 +++--- 4 files changed, 18 insertions(+), 9 deletions(-) diff --git a/pkg/core/core.go b/pkg/core/core.go index a4075a24..f15931d9 100644 --- a/pkg/core/core.go +++ b/pkg/core/core.go @@ -102,7 +102,7 @@ func (comic *ComicIssue) makeEPUB(options *config.Options, images *DownloadResul return err } - if err = e.Write(util.GetPathToFile(dir, comic.Name, comic.IssueNumber, comic.OutputFormat.String(), options.IssueNumberNameOnly)); err != nil { + if err = e.Write(util.GetPathToFile(dir, comic.Name, comic.getIssueNumAndVolume(), comic.OutputFormat.String(), options.IssueNumberNameOnly)); err != nil { return err } @@ -161,7 +161,7 @@ func (comic *ComicIssue) makePDF(options *config.Options, images *DownloadResult return err } - filePath := util.GetPathToFile(dir, comic.Name, comic.IssueNumber, comic.OutputFormat.String(), options.IssueNumberNameOnly) + filePath := util.GetPathToFile(dir, comic.Name, comic.getIssueNumAndVolume(), comic.OutputFormat.String(), options.IssueNumberNameOnly) if err = pdf.OutputFileAndClose(filePath); err != nil { return err } @@ -184,7 +184,7 @@ func (comic *ComicIssue) makeCBRZ(options *config.Options, images *DownloadResul return err } - newName := util.GetPathToFile(dir, comic.Name, comic.IssueNumber, comic.OutputFormat.String(), options.IssueNumberNameOnly) + newName := util.GetPathToFile(dir, comic.Name, comic.getIssueNumAndVolume(), comic.OutputFormat.String(), options.IssueNumberNameOnly) if _, statErr := os.Stat(newName); statErr == nil { if options.Logger != nil { options.Logger.Infof("Skipping %s because it already exists: %s", strings.ToUpper(comic.OutputFormat.String()), newName) @@ -459,6 +459,14 @@ func (comic *ComicIssue) DownloadImages(options *config.Options) (*DownloadResul return &DownloadResult{Dir: dir, FilePaths: paths}, nil } +func (comic *ComicIssue) getIssueNumAndVolume() string { + if comic.Volume == nil { + return fmt.Sprintf("c%s", comic.IssueNumber) + } + + return fmt.Sprintf("v%s c%s", *comic.Volume, comic.IssueNumber) +} + func readExistingImages(dir string) ([]string, error) { entries, err := os.ReadDir(dir) if err != nil { diff --git a/pkg/core/core_test.go b/pkg/core/core_test.go index c9af04cb..2b300805 100644 --- a/pkg/core/core_test.go +++ b/pkg/core/core_test.go @@ -106,7 +106,7 @@ func TestMakeComicPDF(t *testing.T) { require.NoError(t, comic.MakeComic(opts)) - output := filepath.Join(opts.OutputFolder, "comics", comic.Source.Name, comic.Name, "foo-1.pdf") + output := filepath.Join(opts.OutputFolder, "comics", comic.Source.Name, comic.Name, "foo-c1.pdf") require.FileExists(t, output) } @@ -128,7 +128,7 @@ func TestMakeComicEPUB(t *testing.T) { require.NoError(t, comic.MakeComic(opts)) - output := filepath.Join(opts.OutputFolder, "comics", comic.Source.Name, comic.Name, "bar-42.epub") + output := filepath.Join(opts.OutputFolder, "comics", comic.Source.Name, comic.Name, "bar-c42.epub") require.FileExists(t, output) } @@ -154,7 +154,7 @@ func TestMakeComicCBZ(t *testing.T) { require.NoError(t, comic.MakeComic(opts)) - output := filepath.Join(opts.OutputFolder, "comics", comic.Source.Name, comic.Name, "baz-7.cbz") + output := filepath.Join(opts.OutputFolder, "comics", comic.Source.Name, comic.Name, "baz-c7.cbz") require.FileExists(t, output) } diff --git a/pkg/sites/base.go b/pkg/sites/base.go index 596f8605..328d5d1d 100644 --- a/pkg/sites/base.go +++ b/pkg/sites/base.go @@ -8,6 +8,7 @@ type BaseSite interface { // Initialize will initialize the comic struct with the images link Initialize(comic *core.ComicIssue) error + // TODO: remove GetInfo, just use Initialize for getting the comic name and issue number // GetInfo will return the comic name and issue number GetInfo(url string) (string, string, error) diff --git a/pkg/sites/mangadex_test.go b/pkg/sites/mangadex_test.go index 7811d0ca..e77a8487 100644 --- a/pkg/sites/mangadex_test.go +++ b/pkg/sites/mangadex_test.go @@ -246,7 +246,7 @@ func TestMangadexGetInfoEnglish(t *testing.T) { title, chapter, err := md.GetInfo(md.chapterBase + "/chapter-1") require.NoError(t, err) require.Equal(t, "Test Manga", title) - require.Equal(t, "Vol 1 Chapter 1, Start", chapter) + require.Equal(t, "1", chapter) } func TestMangadexGetInfoJapanese(t *testing.T) { @@ -256,7 +256,7 @@ func TestMangadexGetInfoJapanese(t *testing.T) { title, chapter, err := md.GetInfo(md.chapterBase + "/chapter-1") require.NoError(t, err) require.Equal(t, "テスト", title) - require.Equal(t, "Vol 1 Chapter 1, Start", chapter) + require.Equal(t, "1", chapter) } func TestMangadexGetInfoNoCountry(t *testing.T) { @@ -266,7 +266,7 @@ func TestMangadexGetInfoNoCountry(t *testing.T) { title, chapter, err := md.GetInfo(md.chapterBase + "/chapter-1") require.NoError(t, err) require.Equal(t, "テスト", title) - require.Equal(t, "Vol 1 Chapter 1, Start", chapter) + require.Equal(t, "1", chapter) } func TestMangadexInitializeMetadataTagsGenres(t *testing.T) { From aec8851a7ed81e442671f5220ff2506e4766960a Mon Sep 17 00:00:00 2001 From: ProjectDislocate <173194962+ProjectDislocate@users.noreply.github.com> Date: Tue, 21 Apr 2026 18:39:51 -0400 Subject: [PATCH 71/79] feat: apply rate limiting to all requests --- cmd/app/downloader.go | 5 + pkg/core/core.go | 27 ----- pkg/http/limiter.go | 61 ++++++++++++ pkg/http/limiter_test.go | 210 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 276 insertions(+), 27 deletions(-) create mode 100644 pkg/http/limiter.go create mode 100644 pkg/http/limiter_test.go diff --git a/cmd/app/downloader.go b/cmd/app/downloader.go index b94dfb43..08138c95 100644 --- a/cmd/app/downloader.go +++ b/cmd/app/downloader.go @@ -225,6 +225,11 @@ func buildClientOptions(base config.Options) []httpclient.Option { opts = append(opts, httpclient.WithHTTPClient(proxyClient)) } + if base.RequestDelay > 0 || base.RequestDelayJitter > 0 { + limiter := httpclient.NewDelayRateLimiter(base.RequestDelay, base.RequestDelayJitter) + opts = append(opts, httpclient.WithRateLimiter(limiter)) + } + return opts } diff --git a/pkg/core/core.go b/pkg/core/core.go index f15931d9..309658e8 100644 --- a/pkg/core/core.go +++ b/pkg/core/core.go @@ -7,7 +7,6 @@ import ( "fmt" "image" "io" - "math/rand" "net/http" "os" "path" @@ -275,19 +274,6 @@ func (comic *ComicIssue) DownloadImages(options *config.Options) (*DownloadResul outputFormat := util.ImageType(comic.OutputImagesFormat) - requestDelay := options.RequestDelay - requestJitter := options.RequestDelayJitter - if requestDelay < 0 { - requestDelay = 0 - } - if requestJitter < 0 { - requestJitter = 0 - } - if requestDelay == 0 && requestJitter == 0 { - requestDelay = config.DefaultRequestDelay - requestJitter = config.DefaultRequestDelayJitter - } - type downloadJob struct { index int link string @@ -308,8 +294,6 @@ func (comic *ComicIssue) DownloadImages(options *config.Options) (*DownloadResul group, ctx := errgroup.WithContext(ctx) sem := semaphore.NewWeighted(int64(runtime.NumCPU())) var mu sync.Mutex - rng := rand.New(rand.NewSource(time.Now().UnixNano())) - var rngMu sync.Mutex const sniffLimit = 256 for _, job := range jobs { @@ -333,17 +317,6 @@ func (comic *ComicIssue) DownloadImages(options *config.Options) (*DownloadResul reqCtx, cancelReq := context.WithTimeout(ctx, 30*time.Second) defer cancelReq() - sleepDuration := requestDelay - if requestJitter > 0 { - rngMu.Lock() - extra := time.Duration(rng.Int63n(int64(requestJitter))) - rngMu.Unlock() - sleepDuration += extra - } - if sleepDuration > 0 { - time.Sleep(sleepDuration) - } - request, err := client.PrepareRequest(job.link, comic.Source.Name) if err != nil { return err diff --git a/pkg/http/limiter.go b/pkg/http/limiter.go new file mode 100644 index 00000000..b677fc21 --- /dev/null +++ b/pkg/http/limiter.go @@ -0,0 +1,61 @@ +package http + +import ( + "context" + "math/rand" + "sync" + "time" +) + +// DelayRateLimiter implements RateLimiter with configurable base delay and jitter. +// It ensures that the time between consecutive requests is at least baseDelay + random jitter. +type DelayRateLimiter struct { + baseDelay time.Duration + jitter time.Duration + mu sync.Mutex + lastTime time.Time +} + +// NewDelayRateLimiter returns a DelayRateLimiter with the given base delay and jitter. +func NewDelayRateLimiter(baseDelay, jitter time.Duration) *DelayRateLimiter { + return &DelayRateLimiter{ + baseDelay: baseDelay, + jitter: jitter, + } +} + +// Wait implements RateLimiter by enforcing a minimum delay between requests. +// It accounts for the actual time elapsed since the last request and sleeps for the remaining duration. +func (d *DelayRateLimiter) Wait(ctx context.Context) error { + d.mu.Lock() + now := time.Now() + elapsed := now.Sub(d.lastTime) + d.mu.Unlock() + + totalDelay := d.baseDelay + if d.jitter > 0 { + randomJitter := time.Duration(rand.Int63n(int64(d.jitter))) + totalDelay += randomJitter + } + + // Calculate how long we still need to wait + remaining := totalDelay - elapsed + if remaining <= 0 { + // Enough time has passed, just update the timestamp + d.mu.Lock() + d.lastTime = time.Now() + d.mu.Unlock() + return nil + } + + // Sleep for the remaining duration + select { + case <-time.After(remaining): + d.mu.Lock() + d.lastTime = time.Now() + d.mu.Unlock() + return nil + case <-ctx.Done(): + return ctx.Err() + } +} diff --git a/pkg/http/limiter_test.go b/pkg/http/limiter_test.go new file mode 100644 index 00000000..04a64c6b --- /dev/null +++ b/pkg/http/limiter_test.go @@ -0,0 +1,210 @@ +package http + +import ( + "context" + "testing" + "time" +) + +func TestDelayRateLimiterEnforcesBaseDelay(t *testing.T) { + baseDelay := 100 * time.Millisecond + limiter := NewDelayRateLimiter(baseDelay, 0) + + ctx := context.Background() + + // First call initiates the limiter + err := limiter.Wait(ctx) + if err != nil { + t.Fatalf("first wait failed: %v", err) + } + + // Second call should enforce the delay + start := time.Now() + err = limiter.Wait(ctx) + elapsed := time.Since(start) + + if err != nil { + t.Fatalf("second wait failed: %v", err) + } + if elapsed < baseDelay { + t.Errorf("expected to wait at least %v, but waited %v", baseDelay, elapsed) + } +} + +func TestDelayRateLimiterEnforcesDelayBetweenCalls(t *testing.T) { + baseDelay := 100 * time.Millisecond + limiter := NewDelayRateLimiter(baseDelay, 0) + + ctx := context.Background() + + // First call + err := limiter.Wait(ctx) + if err != nil { + t.Fatalf("first wait failed: %v", err) + } + + // Second call should enforce delay from first call + start := time.Now() + err = limiter.Wait(ctx) + elapsed := time.Since(start) + + if err != nil { + t.Fatalf("second wait failed: %v", err) + } + if elapsed < baseDelay { + t.Errorf("expected to wait at least %v between calls, but waited %v", baseDelay, elapsed) + } +} + +func TestDelayRateLimiterAccountsForElapsedTime(t *testing.T) { + baseDelay := 200 * time.Millisecond + limiter := NewDelayRateLimiter(baseDelay, 0) + + ctx := context.Background() + + // First call + err := limiter.Wait(ctx) + if err != nil { + t.Fatalf("first wait failed: %v", err) + } + + // Wait half the base delay manually + time.Sleep(baseDelay / 2) + + // Second call should only wait for the remaining duration + start := time.Now() + err = limiter.Wait(ctx) + remainingElapsed := time.Since(start) + + if err != nil { + t.Fatalf("second wait failed: %v", err) + } + + // Should wait approximately the remaining half (with some tolerance for timing) + expectedMax := baseDelay/2 + 50*time.Millisecond + if remainingElapsed > expectedMax { + t.Errorf("expected to wait at most ~%v for remaining duration, but waited %v", baseDelay/2, remainingElapsed) + } +} + +func TestDelayRateLimiterWithJitter(t *testing.T) { + baseDelay := 50 * time.Millisecond + jitter := 50 * time.Millisecond + + ctx := context.Background() + + // Run multiple times to observe jitter distribution + minWait := time.Duration(1<<63 - 1) + maxWait := time.Duration(0) + + for i := 0; i < 10; i++ { + limiter := NewDelayRateLimiter(baseDelay, jitter) + + // First call initiates the limiter + err := limiter.Wait(ctx) + if err != nil { + t.Fatalf("first wait failed: %v", err) + } + + // Second call experiences the delay + start := time.Now() + err = limiter.Wait(ctx) + elapsed := time.Since(start) + + if err != nil { + t.Fatalf("second wait failed: %v", err) + } + + if elapsed < minWait { + minWait = elapsed + } + if elapsed > maxWait { + maxWait = elapsed + } + } + + // The minimum wait should be at least baseDelay + if minWait < baseDelay { + t.Errorf("minimum wait %v is less than base delay %v", minWait, baseDelay) + } + + // The maximum wait should be at most baseDelay + jitter (with some tolerance) + maxPossible := baseDelay + jitter + 20*time.Millisecond + if maxWait > maxPossible { + t.Errorf("maximum wait %v exceeds expected max %v", maxWait, maxPossible) + } + + // Ideally we see variation due to jitter (but this is probabilistic, so we allow it to be flaky) + variation := maxWait - minWait + if variation > 10*time.Millisecond { + t.Logf("observed jitter variation: %v", variation) + } +} + +func TestDelayRateLimiterRespectsCancellation(t *testing.T) { + baseDelay := 5 * time.Second + limiter := NewDelayRateLimiter(baseDelay, 0) + + ctx := context.Background() + + // First call to initialize the limiter + err := limiter.Wait(ctx) + if err != nil { + t.Fatalf("first wait failed: %v", err) + } + + // Second call with a timeout - should be cancelled before the wait completes + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + err = limiter.Wait(ctx) + elapsed := time.Since(start) + + if err != context.DeadlineExceeded { + t.Errorf("expected context.DeadlineExceeded, got %v", err) + } + if elapsed > 200*time.Millisecond { + t.Errorf("context cancellation should have been respected quickly, but waited %v", elapsed) + } +} + +func TestDelayRateLimiterZeroDelay(t *testing.T) { + limiter := NewDelayRateLimiter(0, 0) + + ctx := context.Background() + start := time.Now() + err := limiter.Wait(ctx) + elapsed := time.Since(start) + + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if elapsed > 50*time.Millisecond { + t.Errorf("zero delay should not cause significant wait, but waited %v", elapsed) + } +} + +func TestDelayRateLimiterConcurrentSafety(t *testing.T) { + baseDelay := 50 * time.Millisecond + limiter := NewDelayRateLimiter(baseDelay, 0) + + ctx := context.Background() + done := make(chan error, 5) + + // Launch multiple goroutines calling Wait concurrently + for i := 0; i < 5; i++ { + go func() { + err := limiter.Wait(ctx) + done <- err + }() + } + + // Collect results + for i := 0; i < 5; i++ { + err := <-done + if err != nil { + t.Errorf("concurrent wait failed: %v", err) + } + } +} From c79c9d6418ffc824d804140a0309674b092725cd Mon Sep 17 00:00:00 2001 From: ProjectDislocate <173194962+ProjectDislocate@users.noreply.github.com> Date: Tue, 21 Apr 2026 18:40:49 -0400 Subject: [PATCH 72/79] feat: make retry an exponential backoff --- pkg/http/client.go | 4 ++- pkg/http/client_test.go | 61 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+), 1 deletion(-) diff --git a/pkg/http/client.go b/pkg/http/client.go index 060e57f4..9d3154dc 100644 --- a/pkg/http/client.go +++ b/pkg/http/client.go @@ -194,8 +194,10 @@ func (c *ComicClient) DoRaw(req *http.Request) (*http.Response, error) { var lastErr error for attempt := 0; attempt < attempts; attempt++ { if attempt > 0 && c.retryWait > 0 { + // Exponential backoff: retryWait * 2^(attempt-1) + backoffDuration := c.retryWait * time.Duration(1< 2.5 { + t.Logf("backoff ratio %.2f is not close to 2x (may be due to timing variations)", ratio) + } +} From 0a813428078ebb975db1db4f04465d8b3349ff39 Mon Sep 17 00:00:00 2001 From: ProjectDislocate <173194962+ProjectDislocate@users.noreply.github.com> Date: Tue, 21 Apr 2026 18:48:54 -0400 Subject: [PATCH 73/79] feat: disambiguate chapter and series title Series titles are now used to set the folder the comics are placed in. Chapter titles are used to set the filename. This change also lets site providers fill in more metadata about a comic before any folder is created. More advanced formatting folder structures are enabled by this, like futher subdividing comics into volume folders. --- pkg/core/core.go | 16 +++--- pkg/core/core_test.go | 24 ++++++--- pkg/core/metadata.go | 2 +- pkg/core/output.go | 4 +- pkg/sites/base.go | 4 -- pkg/sites/comicextra.go | 11 ++++ pkg/sites/common.go | 41 ++++++++++++-- pkg/sites/loader.go | 102 +++++++++++++++++++++-------------- pkg/sites/loader_test.go | 22 ++++---- pkg/sites/mangadex.go | 2 +- pkg/sites/mangareader.go | 13 +++-- pkg/sites/mangatown.go | 11 ++++ pkg/sites/readallcomics.go | 22 +++++++- pkg/sites/readcomiconline.go | 11 ++++ pkg/util/path.go | 12 ++--- pkg/util/path_test.go | 4 +- 16 files changed, 209 insertions(+), 92 deletions(-) diff --git a/pkg/core/core.go b/pkg/core/core.go index 309658e8..ec8351a2 100644 --- a/pkg/core/core.go +++ b/pkg/core/core.go @@ -96,12 +96,12 @@ func (comic *ComicIssue) makeEPUB(options *config.Options, images *DownloadResul } } - dir, err := util.PathSetup(options.CreateDefaultPath, options.OutputFolder, comic.Source.Name, comic.Name) + dir, err := util.PathSetup(options.CreateDefaultPath, options.OutputFolder, comic.Source.Name, comic.SeriesMetadata.Title) if err != nil { return err } - if err = e.Write(util.GetPathToFile(dir, comic.Name, comic.getIssueNumAndVolume(), comic.OutputFormat.String(), options.IssueNumberNameOnly)); err != nil { + if err = e.Write(util.GetPathToFile(dir, comic.ChapterName, comic.GetIssueNumAndVolume(), comic.OutputFormat.String(), options.IssueNumberNameOnly)); err != nil { return err } @@ -155,12 +155,12 @@ func (comic *ComicIssue) makePDF(options *config.Options, images *DownloadResult pdf.ImageOptions(path.Base(fileName), 0, 0, mmWd, mmHt, false, imageOptions, 0, "") } - dir, err := util.PathSetup(options.CreateDefaultPath, options.OutputFolder, comic.Source.Name, comic.Name) + dir, err := util.PathSetup(options.CreateDefaultPath, options.OutputFolder, comic.Source.Name, comic.SeriesMetadata.Title) if err != nil { return err } - filePath := util.GetPathToFile(dir, comic.Name, comic.getIssueNumAndVolume(), comic.OutputFormat.String(), options.IssueNumberNameOnly) + filePath := util.GetPathToFile(dir, comic.ChapterName, comic.GetIssueNumAndVolume(), comic.OutputFormat.String(), options.IssueNumberNameOnly) if err = pdf.OutputFileAndClose(filePath); err != nil { return err } @@ -173,7 +173,7 @@ func (comic *ComicIssue) makePDF(options *config.Options, images *DownloadResult // makeCBRZ will create the CBR/CBZ. func (comic *ComicIssue) makeCBRZ(options *config.Options, images *DownloadResult) error { - dir, err := util.PathSetup(options.CreateDefaultPath, options.OutputFolder, comic.Source.Name, comic.Name) + dir, err := util.PathSetup(options.CreateDefaultPath, options.OutputFolder, comic.Source.Name, comic.SeriesMetadata.Title) if err != nil { return err } @@ -183,7 +183,7 @@ func (comic *ComicIssue) makeCBRZ(options *config.Options, images *DownloadResul return err } - newName := util.GetPathToFile(dir, comic.Name, comic.getIssueNumAndVolume(), comic.OutputFormat.String(), options.IssueNumberNameOnly) + newName := util.GetPathToFile(dir, comic.ChapterName, comic.GetIssueNumAndVolume(), comic.OutputFormat.String(), options.IssueNumberNameOnly) if _, statErr := os.Stat(newName); statErr == nil { if options.Logger != nil { options.Logger.Infof("Skipping %s because it already exists: %s", strings.ToUpper(comic.OutputFormat.String()), newName) @@ -250,7 +250,7 @@ func (comic *ComicIssue) DownloadImages(options *config.Options) (*DownloadResul client := ensureClient(options) - dir, err := util.ImagesPathSetup(options.CreateDefaultPath, options.OutputFolder, comic.Source.Name, comic.Name, options.IssueFolderName, comic.IssueNumber) + dir, err := util.ImagesPathSetup(options.CreateDefaultPath, options.OutputFolder, comic.Source.Name, comic.SeriesMetadata.Title, options.IssueFolderName, comic.IssueNumber) if err != nil { return nil, err } @@ -432,7 +432,7 @@ func (comic *ComicIssue) DownloadImages(options *config.Options) (*DownloadResul return &DownloadResult{Dir: dir, FilePaths: paths}, nil } -func (comic *ComicIssue) getIssueNumAndVolume() string { +func (comic *ComicIssue) GetIssueNumAndVolume() string { if comic.Volume == nil { return fmt.Sprintf("c%s", comic.IssueNumber) } diff --git a/pkg/core/core_test.go b/pkg/core/core_test.go index 2b300805..11b4d69f 100644 --- a/pkg/core/core_test.go +++ b/pkg/core/core_test.go @@ -68,11 +68,14 @@ func TestDownloadImagesCreatesFiles(t *testing.T) { opts := newTestOptions(t, server) comic := &ComicIssue{ - Name: "foo", + ChapterName: "foo", Source: &ComicSource{Name: "test-source", URL: server.URL}, IssueNumber: "1", OutputImagesFormat: "png", ImageLinks: buildLinks(server, 3), + SeriesMetadata: &SeriesMetadata{ + Title: "foo", + }, } result, err := comic.DownloadImages(opts) @@ -96,17 +99,20 @@ func TestMakeComicPDF(t *testing.T) { opts := newTestOptions(t, server) comic := &ComicIssue{ - Name: "foo", + ChapterName: "foo", Source: &ComicSource{Name: "test-source", URL: server.URL}, IssueNumber: "1", OutputFormat: PDF, OutputImagesFormat: "png", ImageLinks: buildLinks(server, 2), + SeriesMetadata: &SeriesMetadata{ + Title: "foo", + }, } require.NoError(t, comic.MakeComic(opts)) - output := filepath.Join(opts.OutputFolder, "comics", comic.Source.Name, comic.Name, "foo-c1.pdf") + output := filepath.Join(opts.OutputFolder, "comics", comic.Source.Name, comic.ChapterName, "foo - c1.pdf") require.FileExists(t, output) } @@ -117,18 +123,20 @@ func TestMakeComicEPUB(t *testing.T) { opts := newTestOptions(t, server) comic := &ComicIssue{ - Name: "bar", + ChapterName: "bar", Source: &ComicSource{Name: "test-source", URL: server.URL}, IssueNumber: "42", OutputFormat: EPUB, OutputImagesFormat: "png", ImageLinks: buildLinks(server, 2), - SeriesMetadata: &SeriesMetadata{}, + SeriesMetadata: &SeriesMetadata{ + Title: "foo", + }, } require.NoError(t, comic.MakeComic(opts)) - output := filepath.Join(opts.OutputFolder, "comics", comic.Source.Name, comic.Name, "bar-c42.epub") + output := filepath.Join(opts.OutputFolder, "comics", comic.Source.Name, comic.SeriesMetadata.Title, "bar - c42.epub") require.FileExists(t, output) } @@ -139,7 +147,7 @@ func TestMakeComicCBZ(t *testing.T) { opts := newTestOptions(t, server) comic := &ComicIssue{ - Name: "baz", + ChapterName: "baz", IssueNumber: "7", OutputFormat: CBZ, @@ -154,7 +162,7 @@ func TestMakeComicCBZ(t *testing.T) { require.NoError(t, comic.MakeComic(opts)) - output := filepath.Join(opts.OutputFolder, "comics", comic.Source.Name, comic.Name, "baz-c7.cbz") + output := filepath.Join(opts.OutputFolder, "comics", comic.Source.Name, comic.SeriesMetadata.Title, "baz - c7.cbz") require.FileExists(t, output) } diff --git a/pkg/core/metadata.go b/pkg/core/metadata.go index 079f13f4..cb7c6ad0 100644 --- a/pkg/core/metadata.go +++ b/pkg/core/metadata.go @@ -83,7 +83,7 @@ type SeriesMetadata struct { // ComicIssue struct contains all the informations about a comic type ComicIssue struct { - Name string // Issue name/title + ChapterName string // Issue name/title IssueNumber string Volume *string diff --git a/pkg/core/output.go b/pkg/core/output.go index 4ede8d69..e96812f5 100644 --- a/pkg/core/output.go +++ b/pkg/core/output.go @@ -48,7 +48,7 @@ func ToComicOutputFormat(format string) (ComicOutputFormat, error) { // makeComicInfoXML generates a ComicInfo.xml file for the given comic issue and saves it to the output directory. It returns the path to the generated ComicInfo.xml file. // Based on the ComicInfo.xml https://anansi-project.github.io/docs/comicinfo/schemas/v2.1 func (comic *ComicIssue) makeComicInfoXML(options *config.Options, images *DownloadResult) (string, error) { - outputDir, err := util.ImagesPathSetup(options.CreateDefaultPath, options.OutputFolder, comic.Source.Name, comic.Name, options.IssueFolderName, comic.IssueNumber) + outputDir, err := util.ImagesPathSetup(options.CreateDefaultPath, options.OutputFolder, comic.Source.Name, comic.ChapterName, options.IssueFolderName, comic.IssueNumber) if err != nil { return "", err } @@ -76,7 +76,7 @@ func (comic *ComicIssue) makeComicInfoXML(options *config.Options, images *Downl comicInfo.CreateElement("Notes").SetText(fmt.Sprintf("Tagged by comics-downloader version %s using info from %s at %s", version.Tag, comic.Source.Name, time.Now().Format(time.RFC3339))) comicInfo.CreateElement("Series").SetText(comic.SeriesMetadata.Title) - comicInfo.CreateElement("Title").SetText(comic.Name) + comicInfo.CreateElement("Title").SetText(comic.ChapterName) localizedTitle := comic.getLocalizedTitle(options) comicInfo.CreateElement("LocalizedSeries").SetText(localizedTitle) diff --git a/pkg/sites/base.go b/pkg/sites/base.go index 328d5d1d..93c7f079 100644 --- a/pkg/sites/base.go +++ b/pkg/sites/base.go @@ -8,10 +8,6 @@ type BaseSite interface { // Initialize will initialize the comic struct with the images link Initialize(comic *core.ComicIssue) error - // TODO: remove GetInfo, just use Initialize for getting the comic name and issue number - // GetInfo will return the comic name and issue number - GetInfo(url string) (string, string, error) - // RetrieveIssueLinks will return the images links of a comic RetrieveIssueLinks() ([]string, error) } diff --git a/pkg/sites/comicextra.go b/pkg/sites/comicextra.go index 2f4cd09f..bb287bea 100644 --- a/pkg/sites/comicextra.go +++ b/pkg/sites/comicextra.go @@ -181,6 +181,17 @@ func (c *Comicextra) GetInfo(url string) (string, string, error) { // Initialize will initialize the comic based // on comicextra.com func (c *Comicextra) Initialize(comic *core.ComicIssue) error { + if comic.SeriesMetadata == nil { + comic.SeriesMetadata = &core.SeriesMetadata{} + } + + parts := util.TrimAndSplitURL(comic.Source.URL) + if len(parts) >= 5 { + comic.ChapterName = parts[3] + comic.SeriesMetadata.Title = parts[3] + comic.IssueNumber = parts[4] + } + links, err := c.retrieveImageLinks(comic) comic.ImageLinks = links diff --git a/pkg/sites/common.go b/pkg/sites/common.go index 90574d2f..564c1e2b 100644 --- a/pkg/sites/common.go +++ b/pkg/sites/common.go @@ -2,6 +2,7 @@ package sites import ( "context" + "fmt" "strings" "github.com/Girbons/comics-downloader/pkg/config" @@ -26,27 +27,48 @@ func MangaKakalotGetInfo(options *config.Options, domain string, url string) (na return "", "", err } - // get chapter name - doc := soup.HTMLParse(res) + return extractMangaKakalotInfo(domain, url, res) +} + +func extractMangaKakalotInfo(domain, url, html string) (name, issueNumber string, err error) { + doc := soup.HTMLParse(html) f := doc.Find("div", "class", breadcrumbClassName(domain)) switch { case strings.Contains(domain, "mangakakalot"): f = f.Find("p") items := f.FindAll("span", "itemprop", "itemListElement") + if len(items) == 0 { + return "", "", fmt.Errorf("could not find mangakakalot breadcrumb entries") + } f = items[len(items)-1] f = f.Find("a").Find("span") case strings.Contains(domain, "manganato"): items := f.FindAll("a", "class", "a-h") + if len(items) == 0 { + return "", "", fmt.Errorf("could not find manganato breadcrumb entries") + } f = items[len(items)-1] + default: + return "", "", fmt.Errorf("unsupported domain for metadata extraction: %s", domain) } + name = f.Text() name, err = regexp2.MustCompile("(Vol\\.[0-9]{1,3} )?(Chapter [0-9]{1,3}(\\.[0-9])?) ?: ", 0).Replace(name, "", 0, 1) if err != nil { return "", "", err } - // parse number from url + parts := util.TrimAndSplitURL(url) - issueNumber = strings.Split(parts[len(parts)-1], "-")[1] + if len(parts) == 0 { + return "", "", fmt.Errorf("invalid URL: %s", url) + } + lastPart := parts[len(parts)-1] + chapterParts := strings.Split(lastPart, "-") + if len(chapterParts) < 2 { + return name, "", nil + } + + issueNumber = chapterParts[1] return name, issueNumber, nil } @@ -59,6 +81,17 @@ func MangaKakalotInitialize(options *config.Options, comic *core.ComicIssue) err return err } + if comic.SeriesMetadata == nil { + comic.SeriesMetadata = &core.SeriesMetadata{} + } + + name, issueNumber, err := extractMangaKakalotInfo(options.SourceName, comic.Source.URL, res) + if err == nil { + comic.ChapterName = name + comic.SeriesMetadata.Title = name + comic.IssueNumber = issueNumber + } + doc := soup.HTMLParse(res) f := doc.Find("div", "class", "container-chapter-reader") var links []string diff --git a/pkg/sites/loader.go b/pkg/sites/loader.go index ea30b99d..adf8d2cd 100644 --- a/pkg/sites/loader.go +++ b/pkg/sites/loader.go @@ -32,6 +32,11 @@ func initializeCollection(issues []string, options *config.Options, base BaseSit return collection, fmt.Errorf("no issues found for URL %q; ensure it points to a specific comic or chapter page", options.URL) } + outputFormat, err := core.ToComicOutputFormat(options.OutputFormat) + if err != nil { + return collection, err + } + var startRange, endRange float64 if options.All && options.IssuesRange != "" { start, end, err := parser.ParseIssuesRange(options.IssuesRange) @@ -43,62 +48,81 @@ func initializeCollection(issues []string, options *config.Options, base BaseSit } for _, url := range issues { - name, issueNumber, err := base.GetInfo(url) - if err != nil { - options.Logger.Errorf("error getting info for url %q: %v", url, err) - continue + comic := &core.ComicIssue{ + OutputFormat: outputFormat, + OutputImagesFormat: options.OutputImagesFormat, + + Source: &core.ComicSource{ + Name: options.SourceName, + URL: url, + }, + SeriesMetadata: &core.SeriesMetadata{ + LocalizedTitle: make(map[string]string), + Description: make(map[string]string), + }, + } + + options.Logger.Debugf("Initializing comic with URL: %s", comic.Source.URL) + if err = base.Initialize(comic); err != nil { + options.Logger.Errorf("error initializing comic for url %q: %v", url, err) + return collection, err + } + + comic.OutputFormat = outputFormat + comic.OutputImagesFormat = options.OutputImagesFormat + if comic.Source == nil { + comic.Source = &core.ComicSource{} + } + comic.Source.Name = options.SourceName + comic.Source.URL = url + + if comic.SeriesMetadata == nil { + comic.SeriesMetadata = &core.SeriesMetadata{} + } + if comic.SeriesMetadata.LocalizedTitle == nil { + comic.SeriesMetadata.LocalizedTitle = make(map[string]string) + } + if comic.SeriesMetadata.Description == nil { + comic.SeriesMetadata.Description = make(map[string]string) } - name = util.Parse(name) + // clean up name + name := util.Parse(comic.ChapterName) + if name == "" { + name = util.Parse(comic.SeriesMetadata.Title) + } if len(options.CustomComicName) > 0 { name = options.CustomComicName } - issueNumber = util.Parse(issueNumber) + if name == "" { + name = util.Parse(options.SourceName) + } - if notInIssuesRange(issueNumber, startRange, endRange) { - options.Logger.Debugf("Skipping issue %q as it is outside the specified range %q", issueNumber, options.IssuesRange) - continue + // attempt to extract issue number for range filtering + issueNumber := util.Parse(comic.IssueNumber) + + comic.ChapterName = name + comic.IssueNumber = issueNumber + if comic.SeriesMetadata.Title == "" { + comic.SeriesMetadata.Title = name } - outputFormat, err := core.ToComicOutputFormat(options.OutputFormat) - if err != nil { - return collection, err + if notInIssuesRange(issueNumber, startRange, endRange) { + options.Logger.Debugf("Skipping issue %q as it is outside the specified range %q", comic.GetIssueNumAndVolume(), options.IssuesRange) + continue } - dir, pathErr := util.PathSetup(options.CreateDefaultPath, options.OutputFolder, options.SourceName, name) + dir, pathErr := util.PathSetup(options.CreateDefaultPath, options.OutputFolder, options.SourceName, comic.SeriesMetadata.Title) if pathErr != nil { return collection, pathErr } - fileName := util.GetPathToFile(dir, name, issueNumber, outputFormat.String(), options.IssueNumberNameOnly) + fileName := util.GetPathToFile(dir, name, comic.GetIssueNumAndVolume(), outputFormat.String(), options.IssueNumberNameOnly) if util.DirectoryOrFileDoesNotExist(fileName) || options.ImagesOnly { - options.Logger.Debugf("Adding issue %q to collection with URL: %s", issueNumber, url) - - comic := &core.ComicIssue{ - Name: name, - IssueNumber: issueNumber, - - OutputFormat: outputFormat, - OutputImagesFormat: options.OutputImagesFormat, - - Source: &core.ComicSource{ - Name: options.SourceName, - URL: url, - }, - SeriesMetadata: &core.SeriesMetadata{ - Title: name, - LocalizedTitle: make(map[string]string), - Description: make(map[string]string), - }, - } - options.Logger.Debugf("Initializing comic with URL: %s", comic.Source.URL) - if err = base.Initialize(comic); err != nil { - options.Logger.Errorf("error initializing comic for url %q: %v", url, err) - return collection, err - } + options.Logger.Debugf("Adding issue %q to collection with URL: %s", comic.GetIssueNumAndVolume(), url) collection = append(collection, comic) } else { - options.Logger.Debugf("Skipping issue %q as it already exists at path: %s", issueNumber, fileName) + options.Logger.Debugf("Skipping issue %q as it already exists at path: %s", comic.GetIssueNumAndVolume(), fileName) } } diff --git a/pkg/sites/loader_test.go b/pkg/sites/loader_test.go index 782b1bd0..095ecddb 100644 --- a/pkg/sites/loader_test.go +++ b/pkg/sites/loader_test.go @@ -26,7 +26,7 @@ func (s *stubSite) Initialize(comic *core.ComicIssue) error { func (s *stubSite) GetInfo(url string) (string, string, error) { if stub, ok := s.comics[url]; ok { - return stub.Name, stub.IssueNumber, nil + return stub.ChapterName, stub.IssueNumber, nil } return "", "", errors.New("missing comic") } @@ -48,9 +48,9 @@ func TestInitializeCollectionFiltersIssues(t *testing.T) { site := &stubSite{ issues: []string{"url-1", "url-2", "url-3"}, comics: map[string]*core.ComicIssue{ - "url-1": {Name: "series", IssueNumber: "issue-1", Source: &core.ComicSource{Name: "test-source", URL: "url-1"}}, - "url-2": {Name: "series", IssueNumber: "issue-2", Source: &core.ComicSource{Name: "test-source", URL: "url-2"}}, - "url-3": {Name: "series", IssueNumber: "issue-3", Source: &core.ComicSource{Name: "test-source", URL: "url-3"}}, + "url-1": {ChapterName: "series", IssueNumber: "issue-1", Source: &core.ComicSource{Name: "test-source", URL: "url-1"}}, + "url-2": {ChapterName: "series", IssueNumber: "issue-2", Source: &core.ComicSource{Name: "test-source", URL: "url-2"}}, + "url-3": {ChapterName: "series", IssueNumber: "issue-3", Source: &core.ComicSource{Name: "test-source", URL: "url-3"}}, }, } @@ -182,8 +182,8 @@ func TestLoadComicFromSourceWithRegistry(t *testing.T) { testSite := &stubSite{ issues: []string{"url-1", "url-2"}, comics: map[string]*core.ComicIssue{ - "url-1": {Name: "test-series", IssueNumber: "1", Source: &core.ComicSource{Name: "test-site", URL: "url-1"}}, - "url-2": {Name: "test-series", IssueNumber: "2", Source: &core.ComicSource{Name: "test-site", URL: "url-2"}}, + "url-1": {ChapterName: "test-series", IssueNumber: "1", Source: &core.ComicSource{Name: "test-site", URL: "url-1"}}, + "url-2": {ChapterName: "test-series", IssueNumber: "2", Source: &core.ComicSource{Name: "test-site", URL: "url-2"}}, }, } @@ -206,9 +206,9 @@ func TestLoadComicFromSourceWithRegistry(t *testing.T) { collection, err := LoadComicFromSource(options) require.NoError(t, err) require.Len(t, collection, 2) - assert.Equal(t, "test-series", collection[0].Name) + assert.Equal(t, "test-series", collection[0].ChapterName) assert.Equal(t, "1", collection[0].IssueNumber) - assert.Equal(t, "test-series", collection[1].Name) + assert.Equal(t, "test-series", collection[1].ChapterName) assert.Equal(t, "2", collection[1].IssueNumber) } @@ -226,7 +226,7 @@ func TestLoadComicFromSourceDisabledSite(t *testing.T) { testSite := &stubSite{ issues: []string{"url-1"}, comics: map[string]*core.ComicIssue{ - "url-1": {Name: "test-series", IssueNumber: "1", Source: &core.ComicSource{Name: "disabled-test-site", URL: "url-1"}}, + "url-1": {ChapterName: "test-series", IssueNumber: "1", Source: &core.ComicSource{Name: "disabled-test-site", URL: "url-1"}}, }, } @@ -262,7 +262,7 @@ func TestLoadComicFromSourcePartialMatch(t *testing.T) { testSite := &stubSite{ issues: []string{"url-1"}, comics: map[string]*core.ComicIssue{ - "url-1": {Name: "my-comic", IssueNumber: "42", Source: &core.ComicSource{Name: "mysite.com", URL: "url-1"}}, + "url-1": {ChapterName: "my-comic", IssueNumber: "42", Source: &core.ComicSource{Name: "mysite.com", URL: "url-1"}}, }, } @@ -285,7 +285,7 @@ func TestLoadComicFromSourcePartialMatch(t *testing.T) { collection, err := LoadComicFromSource(options) require.NoError(t, err) require.Len(t, collection, 1) - assert.Equal(t, "my-comic", collection[0].Name) + assert.Equal(t, "my-comic", collection[0].ChapterName) assert.Equal(t, "42", collection[0].IssueNumber) } diff --git a/pkg/sites/mangadex.go b/pkg/sites/mangadex.go index b9bb4ff1..ee0d9309 100644 --- a/pkg/sites/mangadex.go +++ b/pkg/sites/mangadex.go @@ -634,7 +634,7 @@ func (m *Mangadex) Initialize(comic *core.ComicIssue) error { return err } - // comic.Name = chapter.ChapterTitle // changing the title seems to break path resolving for some reason, probably because the folder has already been created by the time we get to this point, so we just keep the name as is until the metadata system is reworked + comic.ChapterName = chapter.ChapterTitle // changing the title seems to break path resolving for some reason, probably because the folder has already been created by the time we get to this point, so we just keep the name as is until the metadata system is reworked comic.IssueNumber = chapter.ChapterNumber comic.Volume = chapter.Volume comic.LanguageISO = &chapter.TranslatedLanguage diff --git a/pkg/sites/mangareader.go b/pkg/sites/mangareader.go index e14412c1..1d93ab27 100644 --- a/pkg/sites/mangareader.go +++ b/pkg/sites/mangareader.go @@ -135,13 +135,16 @@ func (m *Mangareader) GetInfo(url string) (string, string, error) { // Initialize loads links and metadata from mangareader func (m *Mangareader) Initialize(comic *core.ComicIssue) error { - name, issueNumber, err := m.GetInfo(comic.Source.URL) - if err != nil { - return err + if comic.SeriesMetadata == nil { + comic.SeriesMetadata = &core.SeriesMetadata{} } - comic.Name = name - comic.IssueNumber = issueNumber + parts := util.TrimAndSplitURL(comic.Source.URL) + if len(parts) >= 5 { + comic.ChapterName = parts[3] + comic.SeriesMetadata.Title = parts[3] + comic.IssueNumber = parts[4] + } links, err := m.retrieveImageLinks(comic) comic.ImageLinks = links diff --git a/pkg/sites/mangatown.go b/pkg/sites/mangatown.go index e65c7eda..a8c4785c 100644 --- a/pkg/sites/mangatown.go +++ b/pkg/sites/mangatown.go @@ -159,6 +159,17 @@ func (m *Mangatown) GetInfo(url string) (string, string, error) { // Initialize loads links and metadata from mangatown func (m *Mangatown) Initialize(comic *core.ComicIssue) error { + if comic.SeriesMetadata == nil { + comic.SeriesMetadata = &core.SeriesMetadata{} + } + + parts := util.TrimAndSplitURL(comic.Source.URL) + if len(parts) >= 5 { + comic.ChapterName = parts[4] + comic.SeriesMetadata.Title = parts[4] + comic.IssueNumber = parts[len(parts)-1] + } + links, err := m.retrieveImageLinks(comic) comic.ImageLinks = links diff --git a/pkg/sites/readallcomics.go b/pkg/sites/readallcomics.go index 237f4f83..ab2965d7 100644 --- a/pkg/sites/readallcomics.go +++ b/pkg/sites/readallcomics.go @@ -197,6 +197,10 @@ func (r *Readallcomics) RetrieveIssueLinks() ([]string, error) { // GetInfo extracts the comic info from the given URL. func (r *Readallcomics) GetInfo(url string) (string, string, error) { + return r.extractInfoFromURL(url) +} + +func (r *Readallcomics) extractInfoFromURL(url string) (string, string, error) { parts := util.TrimAndSplitURL(url) lastPart := parts[len(parts)-1] urlParts := strings.Split(lastPart, "-") @@ -425,8 +429,24 @@ func isNumeric(s string) bool { // Initialize prepare the comic instance with links and images. func (r *Readallcomics) Initialize(comic *core.ComicIssue) error { + name, issueNumber, err := r.extractInfoFromURL(comic.Source.URL) + if err != nil { + return err + } + links, err := r.retrieveImageLinks(comic) + if err != nil { + return err + } + + if comic.SeriesMetadata == nil { + comic.SeriesMetadata = &core.SeriesMetadata{} + } + + comic.ChapterName = name + comic.SeriesMetadata.Title = name + comic.IssueNumber = issueNumber comic.ImageLinks = links - return err + return nil } diff --git a/pkg/sites/readcomiconline.go b/pkg/sites/readcomiconline.go index e8baa715..b96451db 100644 --- a/pkg/sites/readcomiconline.go +++ b/pkg/sites/readcomiconline.go @@ -218,6 +218,17 @@ func (c *ReadComicOnline) GetInfo(url string) (string, string, error) { // Initialize will initialize the comic based // on ReadComicOnline.to func (c *ReadComicOnline) Initialize(comic *core.ComicIssue) error { + if comic.SeriesMetadata == nil { + comic.SeriesMetadata = &core.SeriesMetadata{} + } + + parts := util.TrimAndSplitURL(comic.Source.URL) + if len(parts) >= 6 { + comic.ChapterName = parts[4] + comic.SeriesMetadata.Title = parts[4] + comic.IssueNumber = strings.Split(strings.ReplaceAll(parts[5], "Issue-", ""), "?")[0] + } + links, err := c.retrieveImageLinks(comic) comic.ImageLinks = links diff --git a/pkg/util/path.go b/pkg/util/path.go index f308fa23..548533dc 100644 --- a/pkg/util/path.go +++ b/pkg/util/path.go @@ -35,8 +35,8 @@ func TrimNameLength(name string) string { // PathSetup creates the folders where the comic will be saved. // when `createDefaultPath` is false the comic is stored without prepending // the default folder path `comics/source/name/[comic.format]`. -func PathSetup(createDefaultPath bool, outputFolder, source, name string) (string, error) { - path := fmt.Sprintf("%s/comics/%s/%s/", outputFolder, source, strings.TrimSpace(TrimNameLength(name))) +func PathSetup(createDefaultPath bool, outputFolder, source, seriesName string) (string, error) { + path := fmt.Sprintf("%s/comics/%s/%s/", outputFolder, source, strings.TrimSpace(TrimNameLength(seriesName))) if !createDefaultPath { path = fmt.Sprintf("%s/", outputFolder) @@ -48,8 +48,8 @@ func PathSetup(createDefaultPath bool, outputFolder, source, name string) (strin // ImagesPathSetup creates the folders for the images to be saved. // when `createDefaultPath` is false the images are stored without prepending // the default folder path `comics/source/name/[comic.format]`. -func ImagesPathSetup(createDefaultPath bool, outputFolder, source, name, issueFolderName, issueNumber string) (string, error) { - path := fmt.Sprintf("%s/comics/%s/%s/images-%s/", outputFolder, source, strings.TrimSpace(TrimNameLength(name)), strings.TrimSpace(TrimNameLength(issueNumber))) +func ImagesPathSetup(createDefaultPath bool, outputFolder, source, comicName, issueFolderName, issueNumber string) (string, error) { + path := fmt.Sprintf("%s/comics/%s/%s/images-%s/", outputFolder, source, strings.TrimSpace(TrimNameLength(comicName)), strings.TrimSpace(TrimNameLength(issueNumber))) if !createDefaultPath { path = fmt.Sprintf("%s/%s", outputFolder, strings.TrimSpace(TrimNameLength(issueFolderName+issueNumber))) @@ -75,9 +75,9 @@ func DirectoryOrFileDoesNotExist(filePath string) bool { } // GetPathToFile returns the path where the file should be saved. -func GetPathToFile(dir, name, issueNumber, format string, issueNumberOnly bool) string { +func GetPathToFile(dir, comicName, issueNumber, format string, issueNumberOnly bool) string { if issueNumberOnly { return fmt.Sprintf("%s/%s.%s", dir, strings.TrimSpace(TrimNameLength(issueNumber)), format) } - return fmt.Sprintf("%s/%s-%s.%s", dir, strings.TrimSpace(TrimNameLength(name)), strings.TrimSpace(TrimNameLength(issueNumber)), format) + return fmt.Sprintf("%s/%s - %s.%s", dir, strings.TrimSpace(TrimNameLength(comicName)), strings.TrimSpace(TrimNameLength(issueNumber)), format) } diff --git a/pkg/util/path_test.go b/pkg/util/path_test.go index e2cdafe8..b43a1e6c 100644 --- a/pkg/util/path_test.go +++ b/pkg/util/path_test.go @@ -20,7 +20,7 @@ func TestPathSetup(t *testing.T) { func TestGenerateFileName(t *testing.T) { result := GetPathToFile("path/to/something", "comic-name", "invalid_character", "pdf", false) - assert.Equal(t, "path/to/something/comic-name-invalid_character.pdf", result) + assert.Equal(t, "path/to/something/comic-name - invalid_character.pdf", result) result = GetPathToFile("path/to/something", "comic-name", "invalid_character", "pdf", true) assert.Equal(t, "path/to/something/invalid_character.pdf", result) } @@ -87,7 +87,7 @@ func TestGetPathToFileTrimsNameAndIssueNumber(t *testing.T) { longIssueNumber := strings.Repeat("i", NameLength+15) result := GetPathToFile(dir, longName, longIssueNumber, "pdf", false) - expected := fmt.Sprintf("%s/%s-%s.pdf", dir, strings.Repeat("n", NameLength), strings.Repeat("i", NameLength)) + expected := fmt.Sprintf("%s/%s - %s.pdf", dir, strings.Repeat("n", NameLength), strings.Repeat("i", NameLength)) assert.Equal(t, expected, result) resultIssueOnly := GetPathToFile(dir, longName, longIssueNumber, "pdf", true) From dde3330a9043f7501e81020783c279a796bc4722 Mon Sep 17 00:00:00 2001 From: ProjectDislocate <173194962+ProjectDislocate@users.noreply.github.com> Date: Tue, 21 Apr 2026 18:55:53 -0400 Subject: [PATCH 74/79] feat: cleaned names for file system, but leaves metadata alone --- pkg/core/core.go | 19 ++++++++++------- pkg/core/core_test.go | 17 +++++++++------ pkg/core/metadata.go | 4 +++- pkg/core/output.go | 48 ++++++++++++++++++++++++++++++++++++++++++- pkg/sites/loader.go | 28 ++++++++++++------------- pkg/util/path.go | 8 ++++---- pkg/util/path_test.go | 4 ++-- 7 files changed, 93 insertions(+), 35 deletions(-) diff --git a/pkg/core/core.go b/pkg/core/core.go index ec8351a2..c59794af 100644 --- a/pkg/core/core.go +++ b/pkg/core/core.go @@ -96,12 +96,12 @@ func (comic *ComicIssue) makeEPUB(options *config.Options, images *DownloadResul } } - dir, err := util.PathSetup(options.CreateDefaultPath, options.OutputFolder, comic.Source.Name, comic.SeriesMetadata.Title) + outputFilePath, err := comic.GetOutputFilePath(options) if err != nil { return err } - if err = e.Write(util.GetPathToFile(dir, comic.ChapterName, comic.GetIssueNumAndVolume(), comic.OutputFormat.String(), options.IssueNumberNameOnly)); err != nil { + if err = e.Write(outputFilePath); err != nil { return err } @@ -155,12 +155,11 @@ func (comic *ComicIssue) makePDF(options *config.Options, images *DownloadResult pdf.ImageOptions(path.Base(fileName), 0, 0, mmWd, mmHt, false, imageOptions, 0, "") } - dir, err := util.PathSetup(options.CreateDefaultPath, options.OutputFolder, comic.Source.Name, comic.SeriesMetadata.Title) + filePath, err := comic.GetOutputFilePath(options) if err != nil { return err } - filePath := util.GetPathToFile(dir, comic.ChapterName, comic.GetIssueNumAndVolume(), comic.OutputFormat.String(), options.IssueNumberNameOnly) if err = pdf.OutputFileAndClose(filePath); err != nil { return err } @@ -173,7 +172,7 @@ func (comic *ComicIssue) makePDF(options *config.Options, images *DownloadResult // makeCBRZ will create the CBR/CBZ. func (comic *ComicIssue) makeCBRZ(options *config.Options, images *DownloadResult) error { - dir, err := util.PathSetup(options.CreateDefaultPath, options.OutputFolder, comic.Source.Name, comic.SeriesMetadata.Title) + dir, err := comic.GetOutputDir(options) if err != nil { return err } @@ -183,7 +182,13 @@ func (comic *ComicIssue) makeCBRZ(options *config.Options, images *DownloadResul return err } - newName := util.GetPathToFile(dir, comic.ChapterName, comic.GetIssueNumAndVolume(), comic.OutputFormat.String(), options.IssueNumberNameOnly) + newName, err := comic.GetOutputFilePath(options) + if err != nil { + return err + } + + // check if file already exists to avoid creating the archive again + // this is a final sanity check if _, statErr := os.Stat(newName); statErr == nil { if options.Logger != nil { options.Logger.Infof("Skipping %s because it already exists: %s", strings.ToUpper(comic.OutputFormat.String()), newName) @@ -250,7 +255,7 @@ func (comic *ComicIssue) DownloadImages(options *config.Options) (*DownloadResul client := ensureClient(options) - dir, err := util.ImagesPathSetup(options.CreateDefaultPath, options.OutputFolder, comic.Source.Name, comic.SeriesMetadata.Title, options.IssueFolderName, comic.IssueNumber) + dir, err := comic.GetImagesOutputDir(options) if err != nil { return nil, err } diff --git a/pkg/core/core_test.go b/pkg/core/core_test.go index 11b4d69f..f722f956 100644 --- a/pkg/core/core_test.go +++ b/pkg/core/core_test.go @@ -100,19 +100,21 @@ func TestMakeComicPDF(t *testing.T) { comic := &ComicIssue{ ChapterName: "foo", + ChapterNameCleaned: "foo", Source: &ComicSource{Name: "test-source", URL: server.URL}, IssueNumber: "1", OutputFormat: PDF, OutputImagesFormat: "png", ImageLinks: buildLinks(server, 2), SeriesMetadata: &SeriesMetadata{ - Title: "foo", + Title: "foo", + TitleCleaned: "foo", }, } require.NoError(t, comic.MakeComic(opts)) - output := filepath.Join(opts.OutputFolder, "comics", comic.Source.Name, comic.ChapterName, "foo - c1.pdf") + output := filepath.Join(opts.OutputFolder, "comics", comic.Source.Name, comic.ChapterNameCleaned, "foo - c1.pdf") require.FileExists(t, output) } @@ -124,13 +126,15 @@ func TestMakeComicEPUB(t *testing.T) { comic := &ComicIssue{ ChapterName: "bar", + ChapterNameCleaned: "bar", Source: &ComicSource{Name: "test-source", URL: server.URL}, IssueNumber: "42", OutputFormat: EPUB, OutputImagesFormat: "png", ImageLinks: buildLinks(server, 2), SeriesMetadata: &SeriesMetadata{ - Title: "foo", + Title: "foo", + TitleCleaned: "foo", }, } @@ -147,8 +151,8 @@ func TestMakeComicCBZ(t *testing.T) { opts := newTestOptions(t, server) comic := &ComicIssue{ - ChapterName: "baz", - + ChapterName: "baz", + ChapterNameCleaned: "baz", IssueNumber: "7", OutputFormat: CBZ, OutputImagesFormat: "png", @@ -156,7 +160,8 @@ func TestMakeComicCBZ(t *testing.T) { Source: &ComicSource{Name: "test-source", URL: server.URL}, SeriesMetadata: &SeriesMetadata{ - Title: "Baz Series", + Title: "Baz Series", + TitleCleaned: "Baz Series", }, } diff --git a/pkg/core/metadata.go b/pkg/core/metadata.go index cb7c6ad0..0712a54b 100644 --- a/pkg/core/metadata.go +++ b/pkg/core/metadata.go @@ -65,6 +65,7 @@ type SeriesCreator struct { type SeriesMetadata struct { Title string // Series title, should be in the native language of the comic/manga when possible + TitleCleaned string // Cleaned series title with special characters removed, used for file naming LocalizedTitle map[string]string // Map of language code to localized title, e.g. {"en": "One Piece", "jp": "ワンピース"} Description map[string]string // Map of language code to description, e.g. {"en": "A story about pirates...", "jp": "海賊の物語..."} Creators []SeriesCreator @@ -83,7 +84,8 @@ type SeriesMetadata struct { // ComicIssue struct contains all the informations about a comic type ComicIssue struct { - ChapterName string // Issue name/title + ChapterName string // Issue name/title + ChapterNameCleaned string // Cleaned issue name/title with special characters removed, used for file naming IssueNumber string Volume *string diff --git a/pkg/core/output.go b/pkg/core/output.go index e96812f5..e4c55101 100644 --- a/pkg/core/output.go +++ b/pkg/core/output.go @@ -45,10 +45,56 @@ func ToComicOutputFormat(format string) (ComicOutputFormat, error) { } } +// GetOutputDir returns the output directory for the comic issue +func (comic *ComicIssue) GetOutputDir(options *config.Options) (string, error) { + dir, err := util.OutputPathSetup( + options.CreateDefaultPath, + options.OutputFolder, + comic.Source.Name, + comic.SeriesMetadata.TitleCleaned, + ) + if err != nil { + return "", err + } + + return dir, nil +} + +// GetOutputFilePath returns the output file path for the comic issue +func (comic *ComicIssue) GetOutputFilePath(options *config.Options) (string, error) { + dir, err := comic.GetOutputDir(options) + if err != nil { + return "", err + } + + return util.GetPathToFile(dir, + comic.ChapterNameCleaned, + comic.GetIssueNumAndVolume(), + comic.OutputFormat.String(), + options.IssueNumberNameOnly, + ), nil +} + +// GetImagesOutputDir returns the output directory for the comic issue's images +func (comic *ComicIssue) GetImagesOutputDir(options *config.Options) (string, error) { + outputDir, err := util.ImagesPathSetup( + options.CreateDefaultPath, + options.OutputFolder, + comic.Source.Name, + comic.SeriesMetadata.TitleCleaned, + options.IssueFolderName, + comic.IssueNumber, + ) + if err != nil { + return "", err + } + return outputDir, nil +} + // makeComicInfoXML generates a ComicInfo.xml file for the given comic issue and saves it to the output directory. It returns the path to the generated ComicInfo.xml file. // Based on the ComicInfo.xml https://anansi-project.github.io/docs/comicinfo/schemas/v2.1 func (comic *ComicIssue) makeComicInfoXML(options *config.Options, images *DownloadResult) (string, error) { - outputDir, err := util.ImagesPathSetup(options.CreateDefaultPath, options.OutputFolder, comic.Source.Name, comic.ChapterName, options.IssueFolderName, comic.IssueNumber) + outputDir, err := comic.GetOutputDir(options) if err != nil { return "", err } diff --git a/pkg/sites/loader.go b/pkg/sites/loader.go index adf8d2cd..56579eeb 100644 --- a/pkg/sites/loader.go +++ b/pkg/sites/loader.go @@ -26,7 +26,7 @@ func initializeCollection(issues []string, options *config.Options, base BaseSit var collection []*core.ComicIssue // var err error - options.Logger.Debugf("sites: initializing collection for %d issue(s)", len(issues)) + options.Logger.Infof("Initializing metadata for %d issue(s)...", len(issues)) if len(issues) == 0 { return collection, fmt.Errorf("no issues found for URL %q; ensure it points to a specific comic or chapter page", options.URL) @@ -86,25 +86,26 @@ func initializeCollection(issues []string, options *config.Options, base BaseSit comic.SeriesMetadata.Description = make(map[string]string) } - // clean up name - name := util.Parse(comic.ChapterName) - if name == "" { - name = util.Parse(comic.SeriesMetadata.Title) + // clean up cleaedChapterName + cleaedChapterName := util.Parse(comic.ChapterName) + if cleaedChapterName == "" { + cleaedChapterName = util.Parse(comic.SeriesMetadata.Title) } if len(options.CustomComicName) > 0 { - name = options.CustomComicName + cleaedChapterName = options.CustomComicName } - if name == "" { - name = util.Parse(options.SourceName) + cleanedSeriesTitle := util.Parse(comic.SeriesMetadata.Title) + if cleanedSeriesTitle != "" { + comic.SeriesMetadata.TitleCleaned = cleanedSeriesTitle } // attempt to extract issue number for range filtering issueNumber := util.Parse(comic.IssueNumber) - comic.ChapterName = name + comic.ChapterNameCleaned = cleaedChapterName comic.IssueNumber = issueNumber if comic.SeriesMetadata.Title == "" { - comic.SeriesMetadata.Title = name + comic.SeriesMetadata.Title = cleaedChapterName } if notInIssuesRange(issueNumber, startRange, endRange) { @@ -112,11 +113,10 @@ func initializeCollection(issues []string, options *config.Options, base BaseSit continue } - dir, pathErr := util.PathSetup(options.CreateDefaultPath, options.OutputFolder, options.SourceName, comic.SeriesMetadata.Title) - if pathErr != nil { - return collection, pathErr + fileName, err := comic.GetOutputFilePath(options) + if err != nil { + return nil, err } - fileName := util.GetPathToFile(dir, name, comic.GetIssueNumAndVolume(), outputFormat.String(), options.IssueNumberNameOnly) if util.DirectoryOrFileDoesNotExist(fileName) || options.ImagesOnly { options.Logger.Debugf("Adding issue %q to collection with URL: %s", comic.GetIssueNumAndVolume(), url) diff --git a/pkg/util/path.go b/pkg/util/path.go index 548533dc..31cb6f2f 100644 --- a/pkg/util/path.go +++ b/pkg/util/path.go @@ -32,10 +32,10 @@ func TrimNameLength(name string) string { return name } -// PathSetup creates the folders where the comic will be saved. +// OutputPathSetup creates the folders where the comic will be saved. // when `createDefaultPath` is false the comic is stored without prepending // the default folder path `comics/source/name/[comic.format]`. -func PathSetup(createDefaultPath bool, outputFolder, source, seriesName string) (string, error) { +func OutputPathSetup(createDefaultPath bool, outputFolder, source, seriesName string) (string, error) { path := fmt.Sprintf("%s/comics/%s/%s/", outputFolder, source, strings.TrimSpace(TrimNameLength(seriesName))) if !createDefaultPath { @@ -48,8 +48,8 @@ func PathSetup(createDefaultPath bool, outputFolder, source, seriesName string) // ImagesPathSetup creates the folders for the images to be saved. // when `createDefaultPath` is false the images are stored without prepending // the default folder path `comics/source/name/[comic.format]`. -func ImagesPathSetup(createDefaultPath bool, outputFolder, source, comicName, issueFolderName, issueNumber string) (string, error) { - path := fmt.Sprintf("%s/comics/%s/%s/images-%s/", outputFolder, source, strings.TrimSpace(TrimNameLength(comicName)), strings.TrimSpace(TrimNameLength(issueNumber))) +func ImagesPathSetup(createDefaultPath bool, outputFolder, source, seriesName, issueFolderName, issueNumber string) (string, error) { + path := fmt.Sprintf("%s/comics/%s/%s/images-%s/", outputFolder, source, strings.TrimSpace(TrimNameLength(seriesName)), strings.TrimSpace(TrimNameLength(issueNumber))) if !createDefaultPath { path = fmt.Sprintf("%s/%s", outputFolder, strings.TrimSpace(TrimNameLength(issueFolderName+issueNumber))) diff --git a/pkg/util/path_test.go b/pkg/util/path_test.go index b43a1e6c..190337cf 100644 --- a/pkg/util/path_test.go +++ b/pkg/util/path_test.go @@ -11,7 +11,7 @@ import ( ) func TestPathSetup(t *testing.T) { - result, err := PathSetup(true, filepath.Dir(os.Args[0]), "example-source", "comic-name") + result, err := OutputPathSetup(true, filepath.Dir(os.Args[0]), "example-source", "comic-name") assert.Nil(t, err) assert.Contains(t, result, "example-source") @@ -49,7 +49,7 @@ func TestPathSetupTrimsComicName(t *testing.T) { outputFolder := t.TempDir() longComicName := strings.Repeat("comic", 30) - result, err := PathSetup(true, outputFolder, "example-source", longComicName) + result, err := OutputPathSetup(true, outputFolder, "example-source", longComicName) assert.Nil(t, err) assert.Contains(t, result, filepath.Join("comics", "example-source")) From c147f9386937dbd84505493195f5f2b80a199460 Mon Sep 17 00:00:00 2001 From: ProjectDislocate <173194962+ProjectDislocate@users.noreply.github.com> Date: Tue, 21 Apr 2026 18:56:26 -0400 Subject: [PATCH 75/79] chore: easily enable build flag in vscode for go lsp --- .vscode/settings.json | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .vscode/settings.json diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 00000000..52aef911 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,7 @@ +{ + "gopls": { + "build.buildFlags": [ + // "-tags=libjpeg" + ] + }, +} \ No newline at end of file From 5809833b8e538a3dfbb20f45e951d80067c257cb Mon Sep 17 00:00:00 2001 From: ProjectDislocate <173194962+ProjectDislocate@users.noreply.github.com> Date: Tue, 21 Apr 2026 18:57:05 -0400 Subject: [PATCH 76/79] fix: comicinfo.xml not being cleaned up --- pkg/core/output.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/core/output.go b/pkg/core/output.go index e4c55101..f697c726 100644 --- a/pkg/core/output.go +++ b/pkg/core/output.go @@ -94,7 +94,8 @@ func (comic *ComicIssue) GetImagesOutputDir(options *config.Options) (string, er // makeComicInfoXML generates a ComicInfo.xml file for the given comic issue and saves it to the output directory. It returns the path to the generated ComicInfo.xml file. // Based on the ComicInfo.xml https://anansi-project.github.io/docs/comicinfo/schemas/v2.1 func (comic *ComicIssue) makeComicInfoXML(options *config.Options, images *DownloadResult) (string, error) { - outputDir, err := comic.GetOutputDir(options) + // save to images dir as that will be cleaned up after the comic is made + outputDir, err := comic.GetImagesOutputDir(options) if err != nil { return "", err } From 0204585f4361f4bdb450d5bf55f5664cd8ff01bc Mon Sep 17 00:00:00 2001 From: ProjectDislocate <173194962+ProjectDislocate@users.noreply.github.com> Date: Tue, 21 Apr 2026 18:59:22 -0400 Subject: [PATCH 77/79] fix: make timelimit dynamic for download job prevents long running downloads from failing despite more images in queue --- pkg/core/core.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/pkg/core/core.go b/pkg/core/core.go index c59794af..38b63fe3 100644 --- a/pkg/core/core.go +++ b/pkg/core/core.go @@ -293,7 +293,13 @@ func (comic *ComicIssue) DownloadImages(options *config.Options) (*DownloadResul } results := make([]string, len(comic.ImageLinks)) - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + // Calculate timeout dynamically: 5 minutes base + (delay + jitter + download time) per image + // This prevents context deadline exceeded errors on large batches + // Factor in RequestDelay, RequestDelayJitter, and per-image download timeout (30s) + baseTimeout := 5 * time.Minute + timePerImage := options.RequestDelay + options.RequestDelayJitter + 30*time.Second + totalTimeout := baseTimeout + time.Duration(len(jobs))*timePerImage + ctx, cancel := context.WithTimeout(context.Background(), totalTimeout) defer cancel() group, ctx := errgroup.WithContext(ctx) From 25fde5989dba8b665d1a32268d7e60f45683d37e Mon Sep 17 00:00:00 2001 From: ProjectDislocate <173194962+ProjectDislocate@users.noreply.github.com> Date: Tue, 21 Apr 2026 18:59:53 -0400 Subject: [PATCH 78/79] feat: update default user agent --- pkg/http/client.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/http/client.go b/pkg/http/client.go index 9d3154dc..4940fe18 100644 --- a/pkg/http/client.go +++ b/pkg/http/client.go @@ -14,7 +14,7 @@ const ( defaultTimeout = 15 * time.Second defaultRetryCount = 2 defaultRetryWait = 500 * time.Millisecond - defaultUserAgent = "comics-downloader-client" + defaultUserAgent = "comics-downloader-client (https://github.com/Girbons/comics-downloader)" ) // RateLimiter exposes a minimal interface for throttling outgoing requests. From 9fb0d33336cc54731914d05c37c3cbdaa3fbd041 Mon Sep 17 00:00:00 2001 From: ProjectDislocate <173194962+ProjectDislocate@users.noreply.github.com> Date: Tue, 21 Apr 2026 19:00:55 -0400 Subject: [PATCH 79/79] feat: make timeout per request --- pkg/core/core.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/pkg/core/core.go b/pkg/core/core.go index 38b63fe3..1b6009b8 100644 --- a/pkg/core/core.go +++ b/pkg/core/core.go @@ -293,11 +293,17 @@ func (comic *ComicIssue) DownloadImages(options *config.Options) (*DownloadResul } results := make([]string, len(comic.ImageLinks)) + + perRequestTimeout := options.RequestTimeout + if perRequestTimeout <= 10 { + perRequestTimeout = 30 * time.Second + } + // Calculate timeout dynamically: 5 minutes base + (delay + jitter + download time) per image // This prevents context deadline exceeded errors on large batches // Factor in RequestDelay, RequestDelayJitter, and per-image download timeout (30s) baseTimeout := 5 * time.Minute - timePerImage := options.RequestDelay + options.RequestDelayJitter + 30*time.Second + timePerImage := options.RequestDelay + options.RequestDelayJitter + perRequestTimeout totalTimeout := baseTimeout + time.Duration(len(jobs))*timePerImage ctx, cancel := context.WithTimeout(context.Background(), totalTimeout) defer cancel() @@ -325,7 +331,7 @@ func (comic *ComicIssue) DownloadImages(options *config.Options) (*DownloadResul } }() - reqCtx, cancelReq := context.WithTimeout(ctx, 30*time.Second) + reqCtx, cancelReq := context.WithTimeout(ctx, perRequestTimeout) defer cancelReq() request, err := client.PrepareRequest(job.link, comic.Source.Name)