diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 00000000..2fb4bbfe --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,22 @@ +run: + timeout: 5m + tests: false + skip-dirs: + - comics + +linters: + enable: + - errcheck + - govet + - staticcheck + - gosimple + - ineffassign + - typecheck + +linters-settings: + errcheck: + check-type-assertions: true + check-blank: true + +issues: + exclude-use-default: false diff --git a/.travis.yml b/.travis.yml index 9ef54714..9bbde714 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,8 +6,10 @@ go: before_install: - sudo apt-get update - sudo apt install libgl1-mesa-dev xorg-dev + - go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.59.1 - go install github.com/mattn/goveralls@latest script: + - $(go env GOPATH)/bin/golangci-lint run - go test -v ./... - $GOPATH/bin/goveralls -service=travis-ci diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 00000000..18deae9b --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,22 @@ +# Repository Guidelines + +## Project Structure & Module Organization +The project targets Go 1.23+ with a standard layout. `cmd/downloader` hosts the CLI entrypoint, while `cmd/gui` wraps the GUI built with Fyne; both depend on reusable packages under `pkg/`. Key subpackages include `pkg/core` for orchestration, `pkg/sites` for site-specific scrapers, and `pkg/util` for image and filesystem helpers. Supporting flag parsing and other internals live in `internal/`. Docs for contributors live in `docs/`, assets in `img/`, and release artifacts appear under `build/` when generated. + +## Build, Test, and Development Commands +Run `go build -o ./bin/comics-downloader ./cmd/downloader` for a local CLI binary. The GUI can be built with `go build -o ./bin/comics-downloader-gui ./cmd/gui` once Fyne prerequisites are met. Cross-platform artifacts are automated via Make targets such as `make linux-x86-64-build` or the aggregate `make builds`. Use `fyne-cross windows -output comics-downloader-gui-windows.exe ./cmd/gui` when Docker-based cross-compilation is preferred. Static checks run via `make lint` (wraps `golangci-lint run` and matches CI), and `go test -v ./...` should pass before pushing; CI mirrors this command set and reports coverage with Coveralls. + +For sites with aggressive bot protections, you can rotate request headers using `--user-agents "UA1,UA2"` and forward cookies such as `cf_clearance` with `--session-cookie "cf_clearance=...; other=value"` when invoking the CLI. + +## Coding Style & Naming Conventions +Format all Go files with `gofmt` (invoked via `go fmt ./...`) to preserve canonical tab-indented style and import ordering. Follow idiomatic Go naming: exported identifiers use CamelCase, unexported use camelCase, constants are ALL_CAPS only when enumerations require clarity. Keep packages small and cohesive; prefer descriptive filenames like `mangadex.go` aligning with the site handled. Log through the existing Logrus helpers to stay consistent with current output. + +## Testing Guidelines +Place tests alongside implementation in `*_test.go` files, using table-driven cases where practical. Run targeted suites with `go test ./pkg/sites` during feature work, and use `go test -cover ./...` to ensure coverage does not regress—Coveralls tracks the master branch. When scraping new sources, add integration-style tests that stub HTTP calls where possible to keep the suite deterministic. + +## Commit & Pull Request Guidelines +Commit subjects should be concise and imperative (`add`, `fix`, `update`), optionally prefixed with scopes like `fix:` as seen in history. Commit early and keep diffs focused; avoid mixing GUI and CLI changes in one commit. Pull requests must target `master`, include a brief summary of the change, reference related issues, and note how to reproduce or test the update (logs, commands, screenshots for GUI tweaks). Ensure tests pass locally before requesting review. + +## Security & Configuration Tips +- Use `--user-agents` to supply a comma-separated list of desktop/mobile agents; the downloader rotates them per request. +- When Cloudflare or other shields require clearance cookies, reuse your browser session via `--session-cookie` so authenticated requests succeed. diff --git a/Justfile b/Justfile new file mode 100644 index 00000000..d7b0e1cd --- /dev/null +++ b/Justfile @@ -0,0 +1,66 @@ +# List available recipes +default: + @just --list + +# Creates Mac OSX ARM binary +osx-build-arm: + GOOS=darwin go build -o build/comics-downloader-osx-arm ./cmd/downloader + +# Creates Mac OSX x86-64 binary +osx-build-x86-64: + GOOS=darwin GOARCH=amd64 go build -o build/comics-downloader-osx-x86-64 ./cmd/downloader + +# Creates Windows x86-64 binary +windows-x86-64-build: + GOOS=windows GOARCH=amd64 go build -o build/comics-downloader-win-x86-64.exe ./cmd/downloader + +# Creates Windows 386 binary +windows-386-build: + GOOS=windows GOARCH=386 go build -o build/comics-downloader-win-386.exe ./cmd/downloader + +# Creates Linux x86-64 binary +linux-x86-64-build: + GOOS=linux GOARCH=amd64 go build -o build/comics-downloader-linux-x86-64 ./cmd/downloader + +# Creates Linux 386 binary +linux-386-build: + GOOS=linux GOARCH=386 go build -o build/comics-downloader-linux-386 ./cmd/downloader + +# Creates Linux ARM binary +linux-arm-build: + GOOS=linux GOARCH=arm go build -o build/comics-downloader-linux-arm ./cmd/downloader + +# Creates Linux ARM64 binary +linux-arm64-build: + GOOS=linux GOARCH=arm64 go build -o build/comics-downloader-linux-arm64 ./cmd/downloader + +# Creates OSX GUI binary +osx-gui-build: + GOOS=darwin go build -o build/comics-downloader-gui-osx ./cmd/gui + +# Creates Windows GUI executable +windows-gui-build: + fyne-cross windows -output comics-downloader-gui-windows.exe ./cmd/gui + +# Creates Linux GUI executable +linux-gui-build: + fyne-cross linux -output comics-downloader-gui ./cmd/gui + +# Creates executables for OSX/Windows/Linux +builds: linux-386-build linux-arm-build linux-arm64-build linux-x86-64-build osx-build-arm osx-build-x86-64 windows-x86-64-build windows-386-build osx-gui-build linux-gui-build windows-gui-build + +# Remove build artifacts +remove-builds: + rm -rf build/ + +# Run static analysis +lint: + golangci-lint run + +# Run GUI unit tests (headless, no display required) +test-gui: + go test -v ./cmd/gui/... + +# Run the GUI +run-gui: + go run ./cmd/gui diff --git a/Makefile b/Makefile index 2a98b479..38a37e3b 100644 --- a/Makefile +++ b/Makefile @@ -50,3 +50,6 @@ builds: # Creates executables for OSX/Windows/Linux remove-builds: # Remove executables @rm -rf build/ + +lint: # Run static analysis + @golangci-lint run diff --git a/README.md b/README.md index 471f634f..e08434e3 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ - https://comicextra.net/ - https://readallcomics.com/ -- https://readcomiconline.li/ ⚠️ +- https://readcomicsonline.ru/ - https://www.mangareader.tv/ ⚠️ - https://www.mangatown.com/ ⚠️ - https://mangadex.org/ ⚠️ @@ -92,7 +92,7 @@ Usage: | http://www.comicextra.com/ | ✓ | ✗ | ✓ | | http://www.mangatown.com/ | ✓ | ✗ | ✓ | | https://mangadex.org/ | ✓ | ✓ | ✗ | -| https://readcomiconline.li/ | ✓ | ✗ | ✓ | +| https://readcomicsonline.ru/ | ✓ | ✗ | ✓ | | https://www.mangareader.tv/ | ✓ | ✗ | ✓ | | https://www.mangakalot.com/ | ✓ | ✗ | ✓ | | https://www.manganato.com/ | ✓ | ✗ | ✓ | @@ -199,6 +199,16 @@ Default is **jpg**. ./comics-downloader -url=[your url] -images-only -images-format=jpg ``` +### Work Around Site Protection + +Some sources employ rotating fingerprints or require a logged-in session. You can rotate multiple User-Agent strings and forward session cookies collected from your browser: + +```bash +./comics-downloader -url=[your url] \ + --user-agents="Mozilla/5.0 ...,Mozilla/5.0 (Macintosh; ...)" \ + --session-cookie="cf_clearance=...; other=value" +``` + ### Avoid Default Folder Structure The default folder structure that will be created is: diff --git a/TODO.md b/TODO.md new file mode 100644 index 00000000..ec270d3f --- /dev/null +++ b/TODO.md @@ -0,0 +1,32 @@ +# TODO + +## Pending + +### Site Audit — Remove Dead/Broken Sites + +Site status as of 2026-04-07: + +| Site | Domain | Status | Verdict | +|------|--------|--------|---------| +| readcomicsonline.ru | readcomicsonline.ru | ✅ HTTP 200, serves comics | **KEEP** | +| mangadex | api.mangadex.org | ✅ HTTP 200, API responds `pong` | **KEEP** | +| readallcomics | readallcomics.com | ✅ HTTP 200 (with browser UA) | **KEEP** | +| mangatown | mangatown.com | ✅ HTTP 200, site live | **KEEP** | +| comicextra | comicextra.com | ❌ Domain parked (Sedo parking page, not the comic site) | **REMOVE** | +| mangareader | mangareader.tv | ❌ DNS resolution failure — no address associated with hostname | **REMOVE** | +| mangakakalot | mangakakalot.com | ❌ HTTP 522 (Cloudflare origin timeout) — server unreachable | **REMOVE** | +| manganato | manganato.com | ❌ HTTP 522 (Cloudflare origin timeout) — server unreachable | **REMOVE** | + +- [ ] Remove `comicextra` scraper: `pkg/sites/comicextra.go`, `pkg/sites/comicextra_test.go`, `pkg/sites/comicextra_deobfuscate_test.go`, `pkg/sites/deobfuscate.go` (if comicextra-only), update `pkg/sites/loader.go` switch, update README +- [ ] Remove `mangareader` scraper: `pkg/sites/mangareader.go`, `pkg/sites/mangareader_test.go`, update `pkg/sites/loader.go` switch, update README +- [ ] Remove `mangakakalot` scraper: `pkg/sites/mangakakalot.go`, `pkg/sites/mangakakalot_test.go`, update `pkg/sites/loader.go` switch, update README +- [ ] Remove `manganato` scraper: `pkg/sites/manganato.go`, `pkg/sites/manganato_test.go`, update `pkg/sites/loader.go` switch, update README +- [ ] Verify `deobfuscate.go` and `common.go` are not shared with kept sites before deleting +- [ ] Run `go test ./...` and `go build ./...` after removals to confirm clean compile +- [ ] Update README supported-sites table to reflect only active sites + +## Completed +- [x] Add `readcomicsonline.ru` scraper (separate from `readcomiconline.li`) +- [x] Remove `readcomiconline.li` scraper: site uses server-side encrypted obfuscation, permanently broken (`pkg/sites/readcomicsonline.go`) with data-src primary path, JS `var pages` fallback, issue listing, `--all`, `--last` support, and full test coverage +- [x] Convert Makefile to Justfile +- [x] Add GUI unit tests (`cmd/gui/gui_test.go`) and `test-gui` Justfile recipe diff --git a/cmd/app/downloader.go b/cmd/app/downloader.go index 39411709..54a5d16b 100644 --- a/cmd/app/downloader.go +++ b/cmd/app/downloader.go @@ -1,6 +1,8 @@ package app import ( + "context" + "errors" "fmt" "os" "strings" @@ -10,7 +12,7 @@ import ( "github.com/Girbons/comics-downloader/internal/version" "github.com/Girbons/comics-downloader/pkg/config" "github.com/Girbons/comics-downloader/pkg/detector" - "github.com/Girbons/comics-downloader/pkg/http" + httpclient "github.com/Girbons/comics-downloader/pkg/http" "github.com/Girbons/comics-downloader/pkg/sites" "github.com/sirupsen/logrus" ) @@ -22,110 +24,212 @@ var ( Messages = make(chan string) ) -func download(options *config.Options) { - if options.Debug { - options.Logger.SetLevel(logrus.DebugLevel) +// Runner orchestrates a download session based on immutable user input. +type Runner struct { + base config.Options + bindToChannel bool + messages chan string + + loggerFactory func(bind bool, messages chan string) *logger.Logger + clientFactory func() *httpclient.ComicClient + sleep func(time.Duration) +} + +// NewRunner returns a Runner with default factories. +func NewRunner(base config.Options) *Runner { + clientOpts := buildClientOptions(base) + + return &Runner{ + base: base, + loggerFactory: func(bind bool, messages chan string) *logger.Logger { + return logger.NewLogger(bind, messages) + }, + clientFactory: func() *httpclient.ComicClient { + return httpclient.NewComicClient(clientOpts...) + }, + sleep: time.Sleep, + } +} + +// WithChannelBinding enables GUI-friendly logging via the provided channel. +func (r *Runner) WithChannelBinding(messages chan string) { + r.bindToChannel = true + r.messages = messages +} + +func (r *Runner) prepareOptions() config.Options { + opts := r.base + if r.loggerFactory != nil { + opts.Logger = r.loggerFactory(r.bindToChannel, r.messages) + } + if r.clientFactory != nil { + opts.Client = r.clientFactory() + } + return opts +} + +// Run executes a download session, respecting daemon configuration. +func (r *Runner) Run() { + opts := r.prepareOptions() + if opts.Logger == nil { + opts.Logger = logger.NewLogger(false, nil) + } + if opts.Client == nil { + opts.Client = httpclient.NewComicClient() + } + + if opts.URL == "" { + opts.Logger.Error("url parameter is required") + return } - if options.All && options.Last { - options.Last = false - options.Logger.Warning("all and last are selected, all parameter will be used") + if opts.Debug { + opts.Logger.SetLevel(logrus.DebugLevel) + } + + // daemon is started only if `all` or `last` flags are used + if opts.Daemon && (opts.All || opts.Last) { + for { + r.download(opts) + r.sleep(time.Duration(opts.DaemonTimeout) * time.Second) + } + } + + r.download(opts) +} + +func (r *Runner) download(base config.Options) { + opts := base + + if opts.All && opts.Last { + opts.Last = false + opts.Logger.Warning("all and last are selected, all parameter will be used") } // enforce `all` flag when `range` is used. - if options.IssuesRange != "" && !options.All { - options.All = true + if opts.IssuesRange != "" && !opts.All { + opts.All = true } - if options.OutputFolder == "" { + outputFolder := opts.OutputFolder + if outputFolder == "" { dir, err := os.Getwd() if err != nil { - options.Logger.Error("Error determining current directory: %v\n") - options.OutputFolder = "." + opts.Logger.Errorf("Error determining current directory: %v", err) + outputFolder = "." } else { - options.OutputFolder = dir + outputFolder = dir } } - isNewVersionAvailable, newVersionLink, err := version.IsNewAvailable() + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + + isNewVersionAvailable, newVersionLink, err := version.IsNewAvailable(ctx, opts.Client.HTTPClient()) if err != nil { - options.Logger.Error("There was an error while checking for a new comics-downloader version") + if errors.Is(err, version.ErrInvalidSemverTag) { + opts.Logger.Debugf("Skipping version check: %v", err) + } else { + opts.Logger.Errorf("There was an error while checking for a new comics-downloader version: %v", err) + } } if isNewVersionAvailable { - options.Logger.Info(fmt.Sprintf("A new comics-downloader version is available at %s", newVersionLink)) + opts.Logger.Infof("A new comics-downloader version is available at %s", newVersionLink) } - urls := options.URL - - for _, u := range strings.Split(urls, ",") { - if u == "" { + for _, rawURL := range strings.Split(opts.URL, ",") { + trimmedURL := strings.TrimSpace(rawURL) + if trimmedURL == "" { continue } + perURL := opts + perURL.URL = trimmedURL + perURL.OutputFolder = outputFolder + // check if the link is supported - source, check, isDisabled := detector.DetectComic(u) + source, check, isDisabled := detector.DetectComic(trimmedURL) - options.Source = source - options.URL = u + perURL.Source = source if !check { - options.Logger.Error("This site is not supported") + perURL.Logger.Error("This site is not supported") continue } if isDisabled { - options.Logger.Warning("Site currently disabled, please check https://github.com/Girbons/comics-downloader/issues/") + perURL.Logger.Warning("Site currently disabled, please check https://github.com/Girbons/comics-downloader/issues/") continue } - options.Logger.Info("Downloading...") - collection, err := sites.LoadComicFromSource(options) + perURL.Logger.Info("Downloading...") + collection, err := sites.LoadComicFromSource(&perURL) if err != nil { - options.Logger.Error(err.Error()) + perURL.Logger.Error(err.Error()) continue } for _, comic := range collection { - if options.ImagesOnly { - _, err = comic.DownloadImages(options) + if perURL.ImagesOnly { + _, err = comic.DownloadImages(&perURL) } else { - err = comic.MakeComic(options) + err = comic.MakeComic(&perURL) } if err != nil { - options.Logger.Error(err.Error()) + perURL.Logger.Error(err.Error()) } } } } -// GuiRun will start the GUI app +// GuiRun will start the GUI app. func GuiRun(options *config.Options) { AppStatus <- true - options.Logger = logger.NewLogger(true, Messages) - download(options) + runner := NewRunner(*options) + runner.WithChannelBinding(Messages) + runner.Run() AppStatus <- false } -// Run will start the CLI app +// Run will start the CLI app. func Run(options *config.Options) { - options.Logger = logger.NewLogger(false, Messages) - options.Client = http.NewComicClient() + runner := NewRunner(*options) + runner.Run() +} - // link is required - if options.URL == "" { - options.Logger.Error("url parameter is required") - return +func buildClientOptions(base config.Options) []httpclient.Option { + defaultUA := fmt.Sprintf("comics-downloader/%s", version.Tag) + agents := mergeUserAgents(defaultUA, base.UserAgents) + opts := []httpclient.Option{httpclient.WithUserAgents(agents)} + + if strings.TrimSpace(base.SessionCookie) != "" { + opts = append(opts, httpclient.WithHeaders(map[string]string{ + "Cookie": base.SessionCookie, + })) } - // daemon is started only if `all` or `last` flags are used - if options.Daemon && (options.All || options.Last) { - for { - download(options) - time.Sleep(time.Duration(options.DaemonTimeout) * time.Second) + return opts +} + +func mergeUserAgents(defaultAgent string, provided []string) []string { + candidates := append([]string{defaultAgent}, provided...) + seen := make(map[string]struct{}, len(candidates)) + var result []string + for _, candidate := range candidates { + trimmed := strings.TrimSpace(candidate) + if trimmed == "" { + continue + } + if _, ok := seen[trimmed]; ok { + continue } + seen[trimmed] = struct{}{} + result = append(result, trimmed) } - - download(options) + if len(result) == 0 { + result = append(result, defaultAgent) + } + return result } diff --git a/cmd/app/runner_test.go b/cmd/app/runner_test.go new file mode 100644 index 00000000..9797803d --- /dev/null +++ b/cmd/app/runner_test.go @@ -0,0 +1,47 @@ +package app + +import ( + "strings" + "testing" + + "github.com/Girbons/comics-downloader/pkg/config" +) + +func TestRunnerPrepareOptionsProvidesDependencies(t *testing.T) { + runner := NewRunner(config.Options{}) + + opts := runner.prepareOptions() + + if opts.Logger == nil { + t.Fatalf("expected logger to be initialized") + } + + if opts.Client == nil { + t.Fatalf("expected http client to be initialized") + } + + if runner.base.Logger != nil { + t.Fatalf("expected runner base logger to remain nil") + } + if runner.base.Client != nil { + t.Fatalf("expected runner base client to remain nil") + } +} + +func TestRunnerRunRequiresURL(t *testing.T) { + msgs := make(chan string, 1) + + runner := NewRunner(config.Options{}) + runner.WithChannelBinding(msgs) + + runner.Run() + + select { + case msg := <-msgs: + if !strings.Contains(msg, "url parameter is required") { + t.Fatalf("expected error message about missing url, got %q", msg) + } + default: + t.Fatalf("expected an error message to be sent") + } +} diff --git a/cmd/downloader/main.go b/cmd/downloader/main.go index 6e6af4c6..80232d5d 100644 --- a/cmd/downloader/main.go +++ b/cmd/downloader/main.go @@ -4,6 +4,8 @@ import ( "flag" "fmt" "os" + "strings" + "time" "github.com/Girbons/comics-downloader/cmd/app" "github.com/Girbons/comics-downloader/internal/version" @@ -42,6 +44,12 @@ var ( issuesRange string // string to be used for each issue/chapter folder issueFolderName string + // request customization + userAgentsCSV string + sessionCookie string + // throttling + requestDelay time.Duration + requestDelayJitter time.Duration ) func init() { @@ -62,19 +70,16 @@ func init() { flag.StringVar(&outputFolder, "output", "", "Folder where the comics will be saved") flag.StringVar(&issuesRange, "range", "", "Range of issues to download, example 3-9") flag.StringVar(&issueFolderName, "issue-folder-name", "issue-", "Folder name where each issue/chapter will be saved, default 'issue-#'") + flag.StringVar(&userAgentsCSV, "user-agents", "", "Comma-separated list of alternative User-Agent values to rotate per request") + flag.StringVar(&sessionCookie, "session-cookie", "", "Custom Cookie header value (e.g., cf_clearance=...; other=...) for protected sources") + flag.DurationVar(&requestDelay, "request-delay", config.DefaultRequestDelay, "Base delay inserted before downloading each image (e.g. 500ms)") + flag.DurationVar(&requestDelayJitter, "request-delay-jitter", config.DefaultRequestDelayJitter, "Maximum additional random delay added to the base request delay (e.g. 250ms)") flag.IntVar(&daemonTimeout, "daemon-timeout", 600, "DaemonTimeout (seconds), specifies how often the downloader runs") } -func main() { - flag.Parse() - - if versionFlag { - fmt.Println("comics-downloader version", version.Tag) - os.Exit(0) - } - - options := &config.Options{ +func buildOptions() config.Options { + return config.Options{ Debug: debug, All: all, Last: last, @@ -92,7 +97,39 @@ func main() { CreateDefaultPath: createDefaultPath, IssuesRange: issuesRange, IssueFolderName: issueFolderName, + UserAgents: splitAndTrim(userAgentsCSV), + SessionCookie: strings.TrimSpace(sessionCookie), + RequestDelay: requestDelay, + RequestDelayJitter: requestDelayJitter, + } +} + +func main() { + flag.Parse() + + if versionFlag { + fmt.Println("comics-downloader version", version.Tag) + os.Exit(0) } - app.Run(options) + opts := buildOptions() + app.Run(&opts) +} + +func splitAndTrim(csv string) []string { + if strings.TrimSpace(csv) == "" { + return nil + } + parts := strings.Split(csv, ",") + var out []string + for _, part := range parts { + trimmed := strings.TrimSpace(part) + if trimmed != "" { + out = append(out, trimmed) + } + } + if len(out) == 0 { + return nil + } + return out } diff --git a/cmd/downloader/main_test.go b/cmd/downloader/main_test.go new file mode 100644 index 00000000..2a1d50ab --- /dev/null +++ b/cmd/downloader/main_test.go @@ -0,0 +1,124 @@ +package main + +import ( + "testing" +) + +func TestBuildOptionsCopiesGlobals(t *testing.T) { + prev := struct { + debug bool + all bool + last bool + imagesOnly bool + imagesFormat string + country string + forceAspect bool + format string + customComicName string + issueNumberNameOnly bool + url string + outputFolder string + createDefaultPath bool + daemon bool + daemonTimeout int + issuesRange string + issueFolderName string + userAgentsCSV string + sessionCookie string + }{ + debug: debug, + all: all, + last: last, + imagesOnly: imagesOnly, + imagesFormat: imagesFormat, + country: country, + forceAspect: forceAspect, + format: format, + customComicName: customComicName, + issueNumberNameOnly: issueNumberNameOnly, + url: url, + outputFolder: outputFolder, + createDefaultPath: createDefaultPath, + daemon: daemon, + daemonTimeout: daemonTimeout, + issuesRange: issuesRange, + issueFolderName: issueFolderName, + userAgentsCSV: userAgentsCSV, + sessionCookie: sessionCookie, + } + defer func() { + debug = prev.debug + all = prev.all + last = prev.last + imagesOnly = prev.imagesOnly + imagesFormat = prev.imagesFormat + country = prev.country + forceAspect = prev.forceAspect + format = prev.format + customComicName = prev.customComicName + issueNumberNameOnly = prev.issueNumberNameOnly + url = prev.url + outputFolder = prev.outputFolder + createDefaultPath = prev.createDefaultPath + daemon = prev.daemon + daemonTimeout = prev.daemonTimeout + issuesRange = prev.issuesRange + issueFolderName = prev.issueFolderName + userAgentsCSV = prev.userAgentsCSV + sessionCookie = prev.sessionCookie + }() + + debug = true + all = true + last = true + imagesOnly = true + imagesFormat = "png" + country = "jp" + forceAspect = true + format = "epub" + customComicName = "custom" + issueNumberNameOnly = true + url = "http://example.com/comic" + outputFolder = "/tmp/output" + createDefaultPath = false + daemon = true + daemonTimeout = 42 + issuesRange = "1-5" + issueFolderName = "chapter-" + userAgentsCSV = "UA1, UA2 ," + sessionCookie = "cf_clearance=abc123; other=value" + + opts := buildOptions() + + if !opts.Debug || !opts.All || !opts.Last || !opts.ImagesOnly { + t.Fatalf("expected boolean flags to be copied into options: %+v", opts) + } + + if opts.ImagesFormat != "png" || opts.Country != "jp" || opts.Format != "epub" { + t.Fatalf("expected string values to be copied, got %+v", opts) + } + + if opts.CustomComicName != "custom" || opts.URL != "http://example.com/comic" { + t.Fatalf("expected URL and custom name to be copied, got %+v", opts) + } + + if opts.OutputFolder != "/tmp/output" || opts.IssueFolderName != "chapter-" { + t.Fatalf("expected folder values to be copied, got %+v", opts) + } + + if opts.DaemonTimeout != 42 || !opts.Daemon || opts.CreateDefaultPath { + t.Fatalf("expected daemon configuration to be copied, got %+v", opts) + } + + if opts.IssuesRange != "1-5" { + t.Fatalf("expected issues range to be copied, got %q", opts.IssuesRange) + } + + if len(opts.UserAgents) != 2 || opts.UserAgents[0] != "UA1" || opts.UserAgents[1] != "UA2" { + t.Fatalf("expected user agents to be parsed, got %+v", opts.UserAgents) + } + + if opts.SessionCookie != "cf_clearance=abc123; other=value" { + t.Fatalf("expected session cookie to be copied, got %q", opts.SessionCookie) + } +} diff --git a/cmd/gui/gui.go b/cmd/gui/gui.go index 364f4982..c3c44873 100644 --- a/cmd/gui/gui.go +++ b/cmd/gui/gui.go @@ -1,6 +1,8 @@ package main import ( + "strings" + "fyne.io/fyne/widget" downloader "github.com/Girbons/comics-downloader/cmd/app" @@ -44,7 +46,7 @@ func (d *Downloader) Submit() { Debug: d.Debug.Checked, All: d.AllChapters.Checked, Last: d.LastChapter.Checked, - URL: d.URL.Text, + URL: strings.TrimSpace(d.URL.Text), Format: d.Format.Selected, Country: d.Country.Text, ImagesFormat: d.ImagesFormat.Selected, @@ -55,5 +57,5 @@ func (d *Downloader) Submit() { CustomComicName: d.CustomComicName.Text, } - downloader.GuiRun(opts) + go downloader.GuiRun(opts) } diff --git a/cmd/gui/gui_test.go b/cmd/gui/gui_test.go new file mode 100644 index 00000000..eff54108 --- /dev/null +++ b/cmd/gui/gui_test.go @@ -0,0 +1,108 @@ +package main + +import ( + "testing" + + "fyne.io/fyne/test" + "fyne.io/fyne/widget" +) + +// newTestDownloader creates a Downloader wired with real widgets under the +// headless fyne test driver (no display required). +func newTestDownloader() *Downloader { + _ = test.NewApp() // headless driver; safe to call multiple times + return &Downloader{ + URL: widget.NewEntry(), + Country: widget.NewEntry(), + Format: widget.NewRadioGroup([]string{"pdf", "epub", "cbr", "cbz"}, nil), + AllChapters: widget.NewCheck("", nil), + LastChapter: widget.NewCheck("", nil), + ImagesOnly: widget.NewCheck("", nil), + ImagesFormat: widget.NewRadioGroup([]string{"png", "jpg", "img"}, nil), + OutputFolder: widget.NewEntry(), + CreateDefaultPath: widget.NewCheck("", nil), + IssuesRange: widget.NewEntry(), + Debug: widget.NewCheck("", nil), + CustomComicName: widget.NewEntry(), + } +} + +func TestClearURLField(t *testing.T) { + d := newTestDownloader() + d.URL.SetText("https://example.com/comic/1") + d.ClearURLField() + if d.URL.Text != "" { + t.Fatalf("expected empty URL after clear, got %q", d.URL.Text) + } +} + +func TestClearCountryField(t *testing.T) { + d := newTestDownloader() + d.Country.SetText("JP") + d.ClearCountryField() + if d.Country.Text != "" { + t.Fatalf("expected empty country after clear, got %q", d.Country.Text) + } +} + +func TestClearOutputFolderField(t *testing.T) { + d := newTestDownloader() + d.OutputFolder.SetText("/tmp/comics") + d.ClearOutputFolderField() + if d.OutputFolder.Text != "" { + t.Fatalf("expected empty output folder after clear, got %q", d.OutputFolder.Text) + } +} + +func TestDownloaderDefaultFieldValues(t *testing.T) { + d := newTestDownloader() + + if d.URL.Text != "" { + t.Errorf("URL should default to empty, got %q", d.URL.Text) + } + if d.AllChapters.Checked { + t.Errorf("AllChapters should default to unchecked") + } + if d.LastChapter.Checked { + t.Errorf("LastChapter should default to unchecked") + } + if d.ImagesOnly.Checked { + t.Errorf("ImagesOnly should default to unchecked") + } + if d.Debug.Checked { + t.Errorf("Debug should default to unchecked") + } +} + +func TestDownloaderURLTrimmedOnSubmit(t *testing.T) { + // Submit is fire-and-forget (go routine) and calls the real downloader with + // a blank URL, which exits quickly. The main goal here is that wiring + // compiles and the entry text used inside Submit is trimmed. + d := newTestDownloader() + d.URL.SetText(" https://example.com/comic ") + + // We cannot easily intercept the options struct from outside the function, + // but we can verify the widget still holds its raw text and that TrimSpace + // is applied internally (white-box check via direct field read after a + // synthesised Submit would require refactoring; this test guards the + // constructor wiring compiles and the helpers operate correctly). + if d.URL.Text != " https://example.com/comic " { + t.Fatalf("unexpected URL text before submit: %q", d.URL.Text) + } +} + +func TestFormatRadioGroupOptions(t *testing.T) { + d := newTestDownloader() + d.Format.SetSelected("epub") + if d.Format.Selected != "epub" { + t.Fatalf("expected Selected=epub, got %q", d.Format.Selected) + } +} + +func TestImagesFormatRadioGroupOptions(t *testing.T) { + d := newTestDownloader() + d.ImagesFormat.SetSelected("png") + if d.ImagesFormat.Selected != "png" { + t.Fatalf("expected Selected=png, got %q", d.ImagesFormat.Selected) + } +} diff --git a/cmd/gui/main.go b/cmd/gui/main.go index 906f7c2c..f5f658d1 100644 --- a/cmd/gui/main.go +++ b/cmd/gui/main.go @@ -2,6 +2,7 @@ package main import ( "fmt" + "strings" "fyne.io/fyne" "fyne.io/fyne/app" @@ -12,19 +13,54 @@ import ( "github.com/Girbons/comics-downloader/internal/version" ) -func watchLogs(logSection *container.Scroll, box *widget.Box) { - for { - box.Append(widget.NewLabel(<-downloader.Messages)) - logSection.Resize(logSection.Size()) +func watchLogs(logSection *container.Scroll, box *widget.Box, statusLabel *widget.Label) { + for message := range downloader.Messages { + trimmed := strings.TrimSpace(message) + if trimmed == "" { + continue + } + + level := "" + text := trimmed + if parts := strings.SplitN(trimmed, ":", 2); len(parts) == 2 { + level = strings.ToUpper(strings.TrimSpace(parts[0])) + text = strings.TrimSpace(parts[1]) + } + + switch level { + case "INFO", "DEBUG": + if text == "" { + statusLabel.SetText(level) + } else { + statusLabel.SetText(text) + } + case "WARNING", "ERROR": + statusLabel.SetText(trimmed) + box.Append(widget.NewLabel(trimmed)) + logSection.ScrollToBottom() + default: + statusLabel.SetText(trimmed) + } } } -func appStatus(downloadButton *widget.Button) { - for { - if <-downloader.AppStatus { +func appStatus(downloadButton *widget.Button, progress *widget.ProgressBarInfinite, statusLabel *widget.Label) { + for running := range downloader.AppStatus { + if running { downloadButton.Disable() + if !progress.Visible() { + progress.Show() + } + if !progress.Running() { + progress.Start() + } + statusLabel.SetText("Downloading...") } else { + progress.Hide() downloadButton.Enable() + if statusLabel.Text == "Downloading..." { + statusLabel.SetText("Ready") + } } } } @@ -66,6 +102,12 @@ func main() { issuesRange := widget.NewEntry() issuesRange.SetPlaceHolder("1-10") + statusLabel := widget.NewLabel("Ready") + statusLabel.Wrapping = fyne.TextWrapWord + + progress := widget.NewProgressBarInfinite() + progress.Hide() + d := &Downloader{ URL: urlEntry, Country: countryEntry, @@ -116,10 +158,12 @@ func main() { // logSection := widget.NewScrollContainer(box) logSection := container.NewScroll(box) - go watchLogs(logSection, box) - go appStatus(submitButton) + footer := container.NewVBox(statusLabel, progress, buttons) + + go watchLogs(logSection, box, statusLabel) + go appStatus(submitButton, progress, statusLabel) - w.SetContent(fyne.NewContainerWithLayout(layout.NewBorderLayout(form, buttons, nil, nil), form, buttons, logSection)) + w.SetContent(fyne.NewContainerWithLayout(layout.NewBorderLayout(form, footer, nil, nil), form, footer, logSection)) w.Resize(fyne.NewSize(800, 400)) w.ShowAndRun() } diff --git a/docs/dev.md b/docs/dev.md index f903e3fe..37157537 100644 --- a/docs/dev.md +++ b/docs/dev.md @@ -34,3 +34,28 @@ which requires [Docker](https://www.docker.com/get-started). ``` go test -v ./... ``` + +## Lint + +Install golangci-lint once with: + +``` +go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.59.1 +``` + +Then run the full static analysis suite with: + +``` +golangci-lint run +``` + +### Cloudflare / Request Tweaks + +Some sources require browser-like fingerprints. The CLI accepts: + +``` +--user-agents "UA1,UA2" # rotate these agents per request +--session-cookie "cf_clearance=...; other=value" +``` + +Capture values from a working browser session when needed. diff --git a/go.mod b/go.mod index b06c0ac9..4b114139 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,6 @@ require ( github.com/anaskhan96/soup v1.2.5 github.com/bmaupin/go-epub v1.1.0 github.com/dlclark/regexp2 v1.10.0 - github.com/google/go-github v17.0.0+incompatible github.com/jung-kurt/gofpdf v1.16.2 github.com/mholt/archives v0.1.2 github.com/schollz/progressbar/v2 v2.15.0 @@ -37,7 +36,6 @@ require ( github.com/gofrs/uuid v3.1.0+incompatible // indirect github.com/goki/freetype v0.0.0-20181231101311-fa8a33aabaff // indirect github.com/google/go-cmp v0.5.9 // indirect - github.com/google/go-querystring v1.1.0 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect diff --git a/internal/logger/customlogger.go b/internal/logger/customlogger.go index be278a4c..29efdc38 100644 --- a/internal/logger/customlogger.go +++ b/internal/logger/customlogger.go @@ -2,57 +2,99 @@ package logger import ( "fmt" + "strings" + "sync" "github.com/sirupsen/logrus" ) -// Logger is the custom app logger. +// Logger is the custom app logger that can optionally emit messages to a GUI channel. type Logger struct { - l *logrus.Logger + inner *logrus.Logger bindToChannel bool - Messages chan string + messages chan string + channelMu sync.RWMutex } -// NewLogger returns a logger instance +// NewLogger returns a logger instance. func NewLogger(bindToChannel bool, messages chan string) *Logger { + log := logrus.New() + log.SetFormatter(&logrus.TextFormatter{ + DisableTimestamp: true, + DisableLevelTruncation: true, + }) return &Logger{ - l: logrus.New(), - Messages: messages, - bindToChannel: bindToChannel, + inner: log, + bindToChannel: bindToChannel && messages != nil, + messages: messages, } } -// SetLevel set logger level. +// SetLevel sets the logger level. func (logger *Logger) SetLevel(level logrus.Level) { - logger.l.SetLevel(level) + logger.inner.SetLevel(level) } -func (logger *Logger) sendToChannel(msg string) { - if logger.bindToChannel { - logger.Messages <- msg +// Writer exposes the underlying logrus logger for advanced usage. +func (logger *Logger) Writer() *logrus.Logger { + return logger.inner +} + +func (logger *Logger) sendToChannel(level, msg string) { + logger.channelMu.RLock() + defer logger.channelMu.RUnlock() + + if !logger.bindToChannel || logger.messages == nil { + return + } + + formatted := fmt.Sprintf("%s: %s", strings.ToUpper(level), msg) + select { + case logger.messages <- formatted: + default: } } -// Info logs info level log. +// Debug logs at Debug level. +func (logger *Logger) Debug(msg string) { + logger.inner.Debug(msg) + logger.sendToChannel("DEBUG", msg) +} + +// Debugf logs formatted entries at Debug level. +func (logger *Logger) Debugf(format string, args ...interface{}) { + logger.Debug(fmt.Sprintf(format, args...)) +} + +// Info logs at Info level. func (logger *Logger) Info(msg string) { - logger.l.Info(msg) - logger.sendToChannel(fmt.Sprintf("INFO: %s", msg)) + logger.inner.Info(msg) + logger.sendToChannel("INFO", msg) } -// Debug logs debug level log. -func (logger *Logger) Debug(msg string) { - logger.l.Debug(msg) - logger.sendToChannel(fmt.Sprintf("DEBUG: %s", msg)) +// Infof logs formatted entries at Info level. +func (logger *Logger) Infof(format string, args ...interface{}) { + logger.Info(fmt.Sprintf(format, args...)) } -// Warning logs Warning level log. +// Warning logs at Warning level. func (logger *Logger) Warning(msg string) { - logger.l.Warning(msg) - logger.sendToChannel(fmt.Sprintf("WARNING: %s", msg)) + logger.inner.Warning(msg) + logger.sendToChannel("WARNING", msg) } -// Error logs error level log. +// Warningf logs formatted entries at Warning level. +func (logger *Logger) Warningf(format string, args ...interface{}) { + logger.Warning(fmt.Sprintf(format, args...)) +} + +// Error logs at Error level. func (logger *Logger) Error(msg string) { - logger.l.Error(msg) - logger.sendToChannel(fmt.Sprintf("ERROR: %s", msg)) + logger.inner.Error(msg) + logger.sendToChannel("ERROR", msg) +} + +// Errorf logs formatted entries at Error level. +func (logger *Logger) Errorf(format string, args ...interface{}) { + logger.Error(fmt.Sprintf(format, args...)) } diff --git a/internal/logger/customlogger_test.go b/internal/logger/customlogger_test.go new file mode 100644 index 00000000..c22659c8 --- /dev/null +++ b/internal/logger/customlogger_test.go @@ -0,0 +1,46 @@ +package logger + +import ( + "testing" + "time" + + "github.com/sirupsen/logrus" +) + +func TestInfofSendsToChannel(t *testing.T) { + ch := make(chan string, 1) + log := NewLogger(true, ch) + log.SetLevel(logrus.InfoLevel) + + log.Infof("downloaded %d files", 3) + + select { + case msg := <-ch: + if msg != "INFO: downloaded 3 files" { + t.Fatalf("unexpected message %q", msg) + } + default: + t.Fatalf("expected message to be published") + } +} + +func TestChannelSendDoesNotBlock(t *testing.T) { + ch := make(chan string, 1) + log := NewLogger(true, ch) + + log.Info("first") + log.Info("second") // should not block even if buffer is full + + select { + case <-ch: + case <-time.After(100 * time.Millisecond): + t.Fatalf("expected to dequeue at least one message") + } +} + +func TestLoggerHandlesNilChannel(t *testing.T) { + log := NewLogger(false, nil) + log.Debug("noop") + log.Infof("hello %s", "world") + log.Errorf("error %s", "msg") +} diff --git a/internal/version/version.go b/internal/version/version.go index 9baead24..4b46b67f 100644 --- a/internal/version/version.go +++ b/internal/version/version.go @@ -2,33 +2,126 @@ package version import ( "context" + "encoding/json" + "errors" + "fmt" + "log" + "net/http" + "sync" + "time" - "github.com/google/go-github/github" "golang.org/x/mod/semver" ) -// Tag specifies the current release tag. -// It needs to be manually updated. -const Tag = "v0.34-alpha" +const ( + releasesEndpoint = "https://api.github.com/repos/Girbons/comics-downloader/releases?per_page=1" + cacheTTL = time.Minute * 15 +) + +// Tag specifies the current release tag. Overridden at build time via -ldflags. +var Tag = "development" + +// ErrInvalidSemverTag indicates that either the current or latest tag is not valid semantic version. +var ErrInvalidSemverTag = errors.New("invalid semver tag") + +type release struct { + TagName string `json:"tag_name"` + HTMLURL string `json:"html_url"` +} + +type releaseResult struct { + checked time.Time + available bool + link string + err error +} + +var ( + cacheMu sync.Mutex + cachedResult releaseResult +) -// IsNewAvailable will fetch the latest project releases -// and will compare the latest release Tag against the current Tag. -func IsNewAvailable() (bool, string, error) { - ctx := context.Background() - client := github.NewClient(nil) - releases, _, err := client.Repositories.ListReleases(ctx, "Girbons", "comics-downloader", nil) +// IsNewAvailable fetches the latest release information and compares it against the current Tag. +// Results are cached for a short period to avoid repeatedly hammering the API. +func IsNewAvailable(ctx context.Context, client *http.Client) (bool, string, error) { + if ctx == nil { + ctx = context.Background() + } + if client == nil { + client = http.DefaultClient + } + + cacheMu.Lock() + if !cachedResult.checked.IsZero() && time.Since(cachedResult.checked) < cacheTTL { + res := cachedResult + cacheMu.Unlock() + return res.available, res.link, res.err + } + cacheMu.Unlock() + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, releasesEndpoint, nil) if err != nil { return false, "", err } - // Compare returns an integer comparing two versions - // according to semantic version precedence. - result := semver.Compare(Tag, *releases[0].TagName) + resp, err := client.Do(req) + if err != nil { + updateCache(false, "", err) + return false, "", err + } + defer func() { + if closeErr := resp.Body.Close(); closeErr != nil { + log.Printf("version: failed to close release response body: %v", closeErr) + } + }() - // -1 if v < w - if result == -1 { - return true, *releases[0].HTMLURL, err + if resp.StatusCode != http.StatusOK { + err = fmt.Errorf("unexpected status code: %d", resp.StatusCode) + updateCache(false, "", err) + return false, "", err + } + + var releases []release + if err := json.NewDecoder(resp.Body).Decode(&releases); err != nil { + updateCache(false, "", err) + return false, "", err + } + if len(releases) == 0 { + err = fmt.Errorf("no releases found") + updateCache(false, "", err) + return false, "", err + } + latest := releases[0] + + if !semver.IsValid(Tag) || !semver.IsValid(latest.TagName) { + err = fmt.Errorf("%w (current=%q latest=%q)", ErrInvalidSemverTag, Tag, latest.TagName) + updateCache(false, "", err) + return false, "", err + } + + if semver.Compare(Tag, latest.TagName) < 0 { + updateCache(true, latest.HTMLURL, nil) + return true, latest.HTMLURL, nil + } + + updateCache(false, "", nil) + return false, "", nil +} + +func updateCache(available bool, link string, err error) { + cacheMu.Lock() + defer cacheMu.Unlock() + cachedResult = releaseResult{ + checked: time.Now(), + available: available, + link: link, + err: err, } +} - return false, "", err +// ResetCache clears the cached release lookup result. Intended for testing. +func ResetCache() { + cacheMu.Lock() + defer cacheMu.Unlock() + cachedResult = releaseResult{} } diff --git a/internal/version/version_test.go b/internal/version/version_test.go new file mode 100644 index 00000000..ef2ce6d2 --- /dev/null +++ b/internal/version/version_test.go @@ -0,0 +1,121 @@ +package version + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "net/url" + "sync/atomic" + "testing" + "time" +) + +type roundTripFunc func(*http.Request) (*http.Response, error) + +func (f roundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) { + return f(req) +} + +func testHTTPClient(target *url.URL) *http.Client { + return &http.Client{ + Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) { + cloned := req.Clone(req.Context()) + cloned.URL.Scheme = target.Scheme + cloned.URL.Host = target.Host + cloned.Host = target.Host + return http.DefaultTransport.RoundTrip(cloned) + }), + Timeout: time.Second * 5, + } +} + +func TestIsNewAvailableCachesResult(t *testing.T) { + ResetCache() + originalTag := Tag + Tag = "v0.1.0" + defer func() { + Tag = originalTag + ResetCache() + }() + + var hits int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + atomic.AddInt32(&hits, 1) + w.Header().Set("Content-Type", "application/json") + fmt.Fprint(w, `[{"tag_name":"v0.2.0","html_url":"http://example.com/latest"}]`) + })) + defer server.Close() + + serverURL, err := url.Parse(server.URL) + if err != nil { + t.Fatalf("failed to parse test server url: %v", err) + } + + client := testHTTPClient(serverURL) + + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + + available, link, err := IsNewAvailable(ctx, client) + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + if !available { + t.Fatalf("expected new version to be available") + } + if link != "http://example.com/latest" { + t.Fatalf("unexpected link: %s", link) + } + + if got := atomic.LoadInt32(&hits); got != 1 { + t.Fatalf("expected single HTTP call, got %d", got) + } + + ctx2, cancel2 := context.WithTimeout(context.Background(), time.Second) + defer cancel2() + available, _, err = IsNewAvailable(ctx2, client) + if err != nil { + t.Fatalf("expected cached call to succeed, got %v", err) + } + if !available { + t.Fatalf("expected cached call to preserve availability") + } + if got := atomic.LoadInt32(&hits); got != 1 { + t.Fatalf("expected cached call to avoid HTTP, got %d requests", got) + } +} + +func TestIsNewAvailableInvalidSemver(t *testing.T) { + ResetCache() + originalTag := Tag + Tag = "not-a-semver" + defer func() { + Tag = originalTag + ResetCache() + }() + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + fmt.Fprint(w, `[{"tag_name":"v0.2.0","html_url":"http://example.com/latest"}]`) + })) + defer server.Close() + + serverURL, err := url.Parse(server.URL) + if err != nil { + t.Fatalf("failed to parse test server url: %v", err) + } + + client := testHTTPClient(serverURL) + + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + + available, _, err := IsNewAvailable(ctx, client) + if err == nil { + t.Fatalf("expected error for invalid semver") + } + if available { + t.Fatalf("expected availability to be false on error") + } +} diff --git a/pkg/config/options.go b/pkg/config/options.go index b03d5a70..b3d64103 100644 --- a/pkg/config/options.go +++ b/pkg/config/options.go @@ -1,10 +1,19 @@ package config import ( + "time" + "github.com/Girbons/comics-downloader/internal/logger" "github.com/Girbons/comics-downloader/pkg/http" ) +const ( + // DefaultRequestDelay defines the base time to wait between subsequent image requests. + DefaultRequestDelay = 500 * time.Millisecond + // DefaultRequestDelayJitter adds up to this much random extra delay to avoid fixed patterns. + DefaultRequestDelayJitter = 250 * time.Millisecond +) + // Options represents the comics downloader options. type Options struct { Debug bool @@ -25,6 +34,10 @@ type Options struct { Source string IssuesRange string IssueFolderName string + UserAgents []string + SessionCookie string + RequestDelay time.Duration + RequestDelayJitter time.Duration Client *http.ComicClient Logger *logger.Logger diff --git a/pkg/core/core.go b/pkg/core/core.go index 468aef26..e03051e1 100644 --- a/pkg/core/core.go +++ b/pkg/core/core.go @@ -3,18 +3,26 @@ package core import ( "bytes" "context" + "encoding/base64" "fmt" "image" + "io" + "math/rand" + "net/http" "os" "path" + "path/filepath" "runtime" "sort" "strings" + "sync" + "time" "golang.org/x/sync/errgroup" "golang.org/x/sync/semaphore" "github.com/Girbons/comics-downloader/pkg/config" + httpclient "github.com/Girbons/comics-downloader/pkg/http" "github.com/Girbons/comics-downloader/pkg/util" epub "github.com/bmaupin/go-epub" "github.com/jung-kurt/gofpdf" @@ -45,63 +53,46 @@ type Comic struct { ImagesFormat string } -// makeEPUB create the epub file -func (comic *Comic) makeEPUB(options *config.Options) error { - var err error +// DownloadResult captures the outcome of downloading a comic's images. +type DownloadResult struct { + Dir string + FilePaths []string +} - currentDir, err := util.CurrentDir() - if err != nil { - return err +func ensureClient(options *config.Options) *httpclient.ComicClient { + if options.Client == nil { + options.Client = httpclient.NewComicClient() } - // used to check if the epub cover already exists + return options.Client +} + +// makeEPUB creates the epub file. +func (comic *Comic) makeEPUB(options *config.Options, images *DownloadResult) error { isCoverSet := false - // used to add the image in the epub section imgTag := `Cover Image` - // setup a new Epub instance e := epub.NewEpub(comic.IssueNumber) - // set Epub title e.SetTitle(fmt.Sprintf("%s-%s", comic.Name, comic.IssueNumber)) - // check if the author exists for this comic + if comic.Author != "" { e.SetAuthor(comic.Author) } - imagesPath, err := comic.DownloadImages(options) - if err != nil { - return err - } - defer os.RemoveAll(imagesPath) - - files, err := os.ReadDir(imagesPath) - if err != nil { - return err - } - - for _, file := range files { - // add the image to the epub will return a path - imgpath, err := e.AddImage(fmt.Sprintf("%s/%s", imagesPath, file.Name()), "") - if err != nil { + for _, file := range images.FilePaths { + imgpath, err := e.AddImage(file, "") + if err != nil && options.Logger != nil { options.Logger.Error(err.Error()) + continue } - // if the cover is not set use the first image - // otherwise the image will be added as a section if !isCoverSet { isCoverSet = true e.SetCover(imgpath, "") - } else { - _, err = e.AddSection(fmt.Sprintf(imgTag, imgpath), "", "", "") - if err != nil { - options.Logger.Error(err.Error()) - } + continue + } + if _, err := e.AddSection(fmt.Sprintf(imgTag, imgpath), "", "", ""); err != nil && options.Logger != nil { + options.Logger.Error(err.Error()) } } - if err = os.Chdir(currentDir); err != nil { - return err - } - - // get the PathSetup where the file should be saved - // e.g. /www.mangarock.com/comic-name/ dir, err := util.PathSetup(options.CreateDefaultPath, options.OutputFolder, comic.Source, comic.Name) if err != nil { return err @@ -111,50 +102,43 @@ func (comic *Comic) makeEPUB(options *config.Options) error { return err } - options.Logger.Info(fmt.Sprintf("%s %s", strings.ToUpper(comic.Format), DefaultMessage)) - return err + if options.Logger != nil { + options.Logger.Infof("%s %s", strings.ToUpper(comic.Format), DefaultMessage) + } + return nil } -// makePDF create the pdf file -func (comic *Comic) makePDF(options *config.Options) error { - var err error +// makePDF create the pdf file. +func (comic *Comic) makePDF(options *config.Options, images *DownloadResult) error { var mmWd, mmHt float64 const px2mm = 0.2645833333 pdf := gofpdf.New("P", "mm", "A4", "") - imagesPath, err := comic.DownloadImages(options) - if err != nil { - return err - } - - defer os.RemoveAll(imagesPath) - - files, err := os.ReadDir(imagesPath) - if err != nil { - return err - } - imageOptions := gofpdf.ImageOptions{ImageType: util.ImageType(comic.ImagesFormat), ReadDpi: true, AllowNegativePosition: false} - for _, file := range files { + for _, fileName := range images.FilePaths { mmWd = 210.0 mmHt = 297.0 - fileName := fmt.Sprintf("%s/%s", imagesPath, file.Name()) if !options.ForceAspect { img, err := os.Open(fileName) if err != nil { - options.Logger.Error(err.Error()) - } - - defer img.Close() - - im, _, err := image.DecodeConfig(img) - if err != nil { - options.Logger.Error(err.Error()) + if options.Logger != nil { + options.Logger.Error(err.Error()) + } } else { - mmWd = px2mm * float64(im.Width) - mmHt = px2mm * float64(im.Height) + im, _, err := image.DecodeConfig(img) + if closeErr := img.Close(); closeErr != nil && options.Logger != nil { + options.Logger.Errorf("failed to close image %s: %v", fileName, closeErr) + } + if err != nil { + if options.Logger != nil { + options.Logger.Error(err.Error()) + } + } else { + mmWd = px2mm * float64(im.Width) + mmHt = px2mm * float64(im.Height) + } } } pdf.AddPageFormat("P", gofpdf.SizeType{Wd: mmWd, Ht: mmHt}) @@ -164,8 +148,8 @@ func (comic *Comic) makePDF(options *config.Options) error { return err } content := bytes.NewReader(data) - pdf.RegisterImageOptionsReader(file.Name(), imageOptions, content) - pdf.ImageOptions(file.Name(), 0, 0, mmWd, mmHt, false, imageOptions, 0, "") + pdf.RegisterImageOptionsReader(path.Base(fileName), imageOptions, content) + pdf.ImageOptions(path.Base(fileName), 0, 0, mmWd, mmHt, false, imageOptions, 0, "") } dir, err := util.PathSetup(options.CreateDefaultPath, options.OutputFolder, comic.Source, comic.Name) @@ -173,193 +157,309 @@ func (comic *Comic) makePDF(options *config.Options) error { return err } - // Save the pdf file filePath := util.GetPathToFile(dir, comic.Name, comic.IssueNumber, comic.Format, options.IssueNumberNameOnly) if err = pdf.OutputFileAndClose(filePath); err != nil { return err } - options.Logger.Info(fmt.Sprintf("%s %s", strings.ToUpper(comic.Format), DefaultMessage)) - return err -} - -// makeCBRZ will create the CBR/CBZ -func (comic *Comic) makeCBRZ(options *config.Options) error { - var filesToAdd []string - var err error - - imagesPath, err := comic.DownloadImages(options) - if err != nil { - return err - } - defer os.RemoveAll(imagesPath) - - files, err := os.ReadDir(imagesPath) - if err != nil { - return err - } - - for _, file := range files { - filesToAdd = append(filesToAdd, fmt.Sprintf("%s/%s", imagesPath, file.Name())) + if options.Logger != nil { + options.Logger.Infof("%s %s", strings.ToUpper(comic.Format), DefaultMessage) } + return nil +} - // e.g. /www.mangarock.com/comic-name/ +// makeCBRZ will create the CBR/CBZ. +func (comic *Comic) makeCBRZ(options *config.Options, images *DownloadResult) error { dir, err := util.PathSetup(options.CreateDefaultPath, options.OutputFolder, comic.Source, comic.Name) if err != nil { return err } - // the archive must be created as `.zip` then change the extension to `.cbr` or `.cbz`. - zipArchiveName := fmt.Sprintf("%s/%s.zip", dir, comic.IssueNumber) + zipArchiveName := filepath.Join(dir, fmt.Sprintf("%s.zip", comic.IssueNumber)) newName := util.GetPathToFile(dir, comic.Name, comic.IssueNumber, comic.Format, options.IssueNumberNameOnly) - // Create output file out, err := os.Create(zipArchiveName) if err != nil { return err } - defer out.Close() - // Sort files to ensure consistent ordering - sort.Slice(files, func(i, j int) bool { - return files[i].Name() < files[j].Name() - }) - // Map files on disk to their paths in the archive + defer func() { + if out != nil { + if closeErr := out.Close(); closeErr != nil && options.Logger != nil { + options.Logger.Errorf("failed to close archive %s: %v", zipArchiveName, closeErr) + } + } + }() + fileMap := make(map[string]string) - for _, filePath := range filesToAdd { - // Use just the filename (no path) in the archive - fileName := path.Base(filePath) - fileMap[filePath] = fileName + for _, filePath := range images.FilePaths { + fileMap[filePath] = path.Base(filePath) } - // Get files from disk using the archives helper archiveFiles, err := archives.FilesFromDisk(context.Background(), nil, fileMap) if err != nil { return err } - // Create ZIP format (no compression needed for CBZ) format := archives.Zip{} + if err = format.Archive(context.Background(), out, archiveFiles); err != nil { + return err + } - // Create the archive - err = format.Archive(context.Background(), out, archiveFiles) - if err != nil { + if err = out.Close(); err != nil { return err } + out = nil if err = os.Rename(zipArchiveName, newName); err != nil { return err } - options.Logger.Info(fmt.Sprintf("%s %s", strings.ToUpper(comic.Format), DefaultMessage)) + if options.Logger != nil { + options.Logger.Infof("%s %s", strings.ToUpper(comic.Format), DefaultMessage) + } return nil } -// DownloadImages will download the comic/manga images -func (comic *Comic) DownloadImages(options *config.Options) (string, error) { +// DownloadImages will download the comic/manga images. +func (comic *Comic) DownloadImages(options *config.Options) (*DownloadResult, error) { if len(comic.Links) == 0 { - return "", fmt.Errorf("Download failed, no links found for: %s", comic.URLSource) + return nil, fmt.Errorf("download failed, no links found for: %s", comic.URLSource) } - var dir string - var err error + client := ensureClient(options) - dir, err = util.ImagesPathSetup(options.CreateDefaultPath, options.OutputFolder, comic.Source, comic.Name, options.IssueFolderName, comic.IssueNumber) + dir, err := util.ImagesPathSetup(options.CreateDefaultPath, options.OutputFolder, comic.Source, comic.Name, options.IssueFolderName, comic.IssueNumber) if err != nil { - return dir, err + return nil, err } - files, err := os.ReadDir(dir) - if err != nil { - return dir, err + existing, err := readExistingImages(dir) + if err == nil && len(existing) == len(comic.Links) && len(existing) > 0 { + return &DownloadResult{Dir: dir, FilePaths: existing}, nil } - if !util.DirectoryOrFileDoesNotExist(dir) && len(files) == len(comic.Links) { - return dir, err + if err := os.RemoveAll(dir); err != nil { + return nil, err + } + if err := os.MkdirAll(dir, os.ModePerm); err != nil { + return nil, err } + progress := progressbar.NewOptions(len(comic.Links), progressbar.OptionSetRenderBlankState(true)) format := util.ImageType(comic.ImagesFormat) - currentDir, err := util.CurrentDir() - if err != nil { - return dir, err + requestDelay := options.RequestDelay + requestJitter := options.RequestDelayJitter + if requestDelay < 0 { + requestDelay = 0 + } + if requestJitter < 0 { + requestJitter = 0 + } + if requestDelay == 0 && requestJitter == 0 { + requestDelay = config.DefaultRequestDelay + requestJitter = config.DefaultRequestDelayJitter } - // setup the progress bar - bar := progressbar.NewOptions(len(comic.Links), progressbar.OptionSetRenderBlankState(true)) - err = os.Chdir(dir) - if err != nil { - return dir, err + type downloadJob struct { + index int + link string } - g := new(errgroup.Group) + jobs := make([]downloadJob, 0, len(comic.Links)) + for idx, link := range comic.Links { + if strings.TrimSpace(link) == "" { + continue + } + jobs = append(jobs, downloadJob{index: idx, link: link}) + } - maxWorkers := int64(runtime.NumCPU()) - sem := semaphore.NewWeighted(maxWorkers) - ctx := context.Background() + results := make([]string, len(comic.Links)) + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() - defer sem.Acquire(ctx, maxWorkers) - for i, link := range comic.Links { - link, i := link, i - sem.Acquire(ctx, 1) + group, ctx := errgroup.WithContext(ctx) + sem := semaphore.NewWeighted(int64(runtime.NumCPU())) + var mu sync.Mutex + rng := rand.New(rand.NewSource(time.Now().UnixNano())) + var rngMu sync.Mutex + const sniffLimit = 256 - if link == "" { - continue + for _, job := range jobs { + job := job + if err := sem.Acquire(ctx, 1); err != nil { + return nil, err } - - g.Go(func() error { + group.Go(func() error { defer sem.Release(1) - rsp, err := options.Client.Get(link, comic.Source) + defer func() { + if progressErr := progress.Add(1); progressErr != nil && options.Logger != nil { + options.Logger.Error(progressErr.Error()) + } + }() + + reqCtx, cancelReq := context.WithTimeout(ctx, 30*time.Second) + defer cancelReq() + + sleepDuration := requestDelay + if requestJitter > 0 { + rngMu.Lock() + extra := time.Duration(rng.Int63n(int64(requestJitter))) + rngMu.Unlock() + sleepDuration += extra + } + if sleepDuration > 0 { + time.Sleep(sleepDuration) + } + + request, err := client.PrepareRequest(job.link, comic.Source) if err != nil { return err } - defer rsp.Body.Close() + request = request.WithContext(reqCtx) - imgName := fmt.Sprintf("%04d-image.%s", i, format) - imgFile, err := os.Create(imgName) + response, err := client.Do(request) if err != nil { return err } - defer imgFile.Close() + defer func() { + if closeErr := response.Body.Close(); closeErr != nil { + if options.Logger != nil { + options.Logger.Errorf("failed to close response body for %s: %v", job.link, closeErr) + } + } + }() + + if response.StatusCode < http.StatusOK || response.StatusCode >= http.StatusMultipleChoices { + if options.Logger != nil { + options.Logger.Errorf("There was an error while downloading image number: %d - comic issue: %s (status code: %d)", job.index, comic.IssueNumber, response.StatusCode) + } + return nil + } - isWebp := strings.HasSuffix(link, ".webp") - err = util.SaveImage(imgFile, rsp.Body, format, isWebp) + data, err := io.ReadAll(response.Body) if err != nil { - msgError := fmt.Sprintf("There was an error while downloading image number: %d - comic issue: %s", i, comic.IssueNumber) - options.Logger.Error(msgError) - os.Remove(imgName) + if options.Logger != nil { + options.Logger.Errorf("Failed reading image number: %d - comic issue: %s (%v)", job.index, comic.IssueNumber, err) + } + return nil } - if barErr := bar.Add(1); barErr != nil { - options.Logger.Error(barErr.Error()) + if len(data) == 0 { + if options.Logger != nil { + options.Logger.Errorf("Image number: %d - comic issue: %s returned an empty response body", job.index, comic.IssueNumber) + } + return nil } + + contentType := strings.ToLower(strings.TrimSpace(response.Header.Get("Content-Type"))) + if contentType == "" { + sniffLen := len(data) + if sniffLen > sniffLimit { + sniffLen = sniffLimit + } + contentType = strings.ToLower(http.DetectContentType(data[:sniffLen])) + } + + isWebp := strings.HasSuffix(strings.ToLower(job.link), ".webp") || strings.Contains(contentType, "image/webp") + if options.Logger != nil && contentType != "" && !strings.HasPrefix(contentType, "image/") { + reportLen := len(data) + if reportLen > sniffLimit { + reportLen = sniffLimit + } + snippet := base64.StdEncoding.EncodeToString(data[:reportLen]) + options.Logger.Errorf("Unexpected content type '%s' while downloading image number: %d - url: %s (bytes=%d, snippet_base64=%s)", contentType, job.index, job.link, len(data), snippet) + } + + fileName := fmt.Sprintf("%04d-image.%s", job.index, format) + targetPath := filepath.Join(dir, fileName) + imgFile, err := os.Create(targetPath) + if err != nil { + return err + } + + reader := bytes.NewReader(data) + if err := util.SaveImage(imgFile, reader, format, isWebp); err != nil { + if options.Logger != nil { + reportLen := len(data) + if reportLen > sniffLimit { + reportLen = sniffLimit + } + snippet := base64.StdEncoding.EncodeToString(data[:reportLen]) + options.Logger.Errorf("There was an error while downloading image number: %d - comic issue: %s (%v) (content-type=%s bytes=%d snippet_base64=%s)", job.index, comic.IssueNumber, err, contentType, len(data), snippet) + } + if closeErr := imgFile.Close(); closeErr != nil && options.Logger != nil { + options.Logger.Errorf("failed to close image file %s: %v", targetPath, closeErr) + } + if removeErr := os.Remove(targetPath); removeErr != nil && options.Logger != nil { + options.Logger.Errorf("failed to remove incomplete image %s: %v", targetPath, removeErr) + } + } else { + if closeErr := imgFile.Close(); closeErr != nil && options.Logger != nil { + options.Logger.Errorf("failed to close image file %s: %v", targetPath, closeErr) + } + mu.Lock() + results[job.index] = targetPath + mu.Unlock() + } + return nil }) } - if err := g.Wait(); err != nil { - return dir, err + if err := group.Wait(); err != nil { + return nil, err } - err = os.Chdir(currentDir) + paths := filterEmpty(results) + sort.Strings(paths) + return &DownloadResult{Dir: dir, FilePaths: paths}, nil +} + +func readExistingImages(dir string) ([]string, error) { + entries, err := os.ReadDir(dir) if err != nil { - return dir, err + return nil, err + } + var files []string + for _, entry := range entries { + if entry.IsDir() { + continue + } + files = append(files, filepath.Join(dir, entry.Name())) } + sort.Strings(files) + return files, nil +} - return dir, err +func filterEmpty(items []string) []string { + var filtered []string + for _, item := range items { + if item != "" { + filtered = append(filtered, item) + } + } + return filtered } // MakeComic will create the file based on the output format selected. func (comic *Comic) MakeComic(options *config.Options) error { - var err error + result, err := comic.DownloadImages(options) + if err != nil { + return err + } + defer func() { + if err := os.RemoveAll(result.Dir); err != nil && options.Logger != nil { + options.Logger.Errorf("failed to remove temporary directory %s: %v", result.Dir, err) + } + }() switch comic.Format { case EPUB: - err = comic.makeEPUB(options) + return comic.makeEPUB(options, result) case CBR, CBZ: - err = comic.makeCBRZ(options) + return comic.makeCBRZ(options, result) default: - err = comic.makePDF(options) + return comic.makePDF(options, result) } - - return err } diff --git a/pkg/core/core_test.go b/pkg/core/core_test.go index e87e73dc..0d2e060d 100644 --- a/pkg/core/core_test.go +++ b/pkg/core/core_test.go @@ -1,7 +1,13 @@ package core import ( - "fmt" + "bytes" + "encoding/base64" + "image" + _ "image/png" + "io" + "net/http" + "net/http/httptest" "os" "path/filepath" "testing" @@ -9,169 +15,150 @@ import ( "github.com/Girbons/comics-downloader/internal/logger" "github.com/Girbons/comics-downloader/pkg/config" - "github.com/Girbons/comics-downloader/pkg/http" - "github.com/stretchr/testify/assert" + httpclient "github.com/Girbons/comics-downloader/pkg/http" + "github.com/stretchr/testify/require" ) -func exists(f string) bool { - _, err := os.Stat(f) - if os.IsNotExist(err) { - return false +var samplePNG = func() []byte { + data, _ := base64.StdEncoding.DecodeString( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADElEQVR4nGP4//8/AAX+Av4N70a4AAAAAElFTkSuQmCC", + ) + return data +}() + +func newImageServer() *httptest.Server { + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "image/png") + _, _ = w.Write(samplePNG) + })) +} + +func newTestOptions(t *testing.T, server *httptest.Server) *config.Options { + t.Helper() + + client := httpclient.NewComicClient( + httpclient.WithHTTPClient(server.Client()), + httpclient.WithRetry(0, 0), + ) + + return &config.Options{ + OutputFolder: t.TempDir(), + CreateDefaultPath: true, + Debug: false, + Logger: logger.NewLogger(false, nil), + Client: client, + IssueFolderName: "issue-", + RequestDelay: time.Nanosecond, + RequestDelayJitter: 0, } - return err == nil } -func TestNewComic(t *testing.T) { - comic := new(Comic) - // links - links := []string{"foo.example.com"} +func buildLinks(server *httptest.Server, count int) []string { + links := make([]string, count) + for i := 0; i < count; i++ { + links[i] = server.URL + "/img.png" + } + return links +} - comic.Name = "foo" - comic.IssueNumber = "2" - comic.Links = links - comic.Source = "bar" - comic.ImagesFormat = "png" +func TestDownloadImagesCreatesFiles(t *testing.T) { + server := newImageServer() + defer server.Close() - assert.Equal(t, "foo", comic.Name) - assert.Equal(t, "2", comic.IssueNumber) - assert.Equal(t, "bar", comic.Source) + opts := newTestOptions(t, server) - assert.Equal(t, 1, len(comic.Links)) + comic := &Comic{ + Name: "foo", + Source: "test-source", + IssueNumber: "1", + ImagesFormat: "png", + Links: buildLinks(server, 3), + } + + result, err := comic.DownloadImages(opts) + require.NoError(t, err) + require.Len(t, result.FilePaths, 3) + + for _, file := range result.FilePaths { + data, err := readFile(file) + require.NoError(t, err) + require.NotEmpty(t, data) + _, format, err := image.Decode(bytes.NewReader(data)) + require.NoError(t, err) + require.Equal(t, "png", format) + } } func TestMakeComicPDF(t *testing.T) { - comic := new(Comic) - - comic.Name = "foo" - comic.Format = "pdf" - comic.IssueNumber = "example-chapter-1" - comic.Links = []string{"https://via.placeholder.com/150", "https://via.placeholder.com/150", "https://via.placeholder.com/150"} - comic.ImagesFormat = "png" - - opt := &config.Options{ - OutputFolder: filepath.Dir(os.Args[0]), - CreateDefaultPath: true, - Debug: false, - Logger: logger.NewLogger(false, make(chan string)), - Client: http.NewComicClient(), + server := newImageServer() + defer server.Close() + + opts := newTestOptions(t, server) + + comic := &Comic{ + Name: "foo", + Source: "test-source", + IssueNumber: "1", + Format: PDF, + ImagesFormat: "png", + Links: buildLinks(server, 2), } - time.Sleep(5 * time.Second) - err := comic.MakeComic(opt) - assert.Nil(t, err) - dir, _ := filepath.Abs(fmt.Sprintf("%s/%s/%s/%s/", filepath.Dir(os.Args[0]), "comics", "foo", "foo-example-chapter-1.pdf")) - assert.True(t, exists(dir)) + require.NoError(t, comic.MakeComic(opts)) + + output := filepath.Join(opts.OutputFolder, "comics", comic.Source, comic.Name, "foo-1.pdf") + require.FileExists(t, output) } func TestMakeComicEPUB(t *testing.T) { - comic := new(Comic) - - comic.Name = "foo" - comic.Format = "epub" - comic.IssueNumber = "example-chapter-1" - comic.Author = "author" - comic.ImagesFormat = "png" - - comic.Links = []string{"https://via.placeholder.com/150", "https://via.placeholder.com/150", "https://via.placeholder.com/150"} - - opt := &config.Options{ - OutputFolder: filepath.Dir(os.Args[0]), - CreateDefaultPath: true, - Debug: false, - Logger: logger.NewLogger(false, make(chan string)), - Client: http.NewComicClient(), + server := newImageServer() + defer server.Close() + + opts := newTestOptions(t, server) + + comic := &Comic{ + Name: "bar", + Source: "test-source", + IssueNumber: "42", + Author: "Author", + Format: EPUB, + ImagesFormat: "png", + Links: buildLinks(server, 2), } - time.Sleep(10 * time.Second) - err := comic.MakeComic(opt) - assert.Nil(t, err) + require.NoError(t, comic.MakeComic(opts)) - dir, _ := filepath.Abs(fmt.Sprintf("%s/%s/%s/%s/", filepath.Dir(os.Args[0]), "comics", "foo", "foo-example-chapter-1.epub")) - assert.True(t, exists(dir)) + output := filepath.Join(opts.OutputFolder, "comics", comic.Source, comic.Name, "bar-42.epub") + require.FileExists(t, output) } -func TestDownloadImagesPNGFormat(t *testing.T) { - comic := new(Comic) - - comic.Name = "foo-png" - comic.Source = "fake" - comic.IssueNumber = "example-chapter-1" - comic.Links = []string{"https://via.placeholder.com/150", "https://via.placeholder.com/150", "https://via.placeholder.com/150"} - comic.ImagesFormat = "png" - - opt := &config.Options{ - OutputFolder: filepath.Dir(os.Args[0]), - Debug: false, - CreateDefaultPath: true, - Logger: logger.NewLogger(false, make(chan string)), - Client: http.NewComicClient(), - } - time.Sleep(10 * time.Second) - _, err := comic.DownloadImages(opt) - assert.Nil(t, err) -} +func TestMakeComicCBZ(t *testing.T) { + server := newImageServer() + defer server.Close() -func TestDownloadImagesJPGFormat(t *testing.T) { - comic := new(Comic) - - comic.Name = "foo-jpg" - comic.Source = "fake" - comic.IssueNumber = "example-chapter-1" - comic.Links = []string{"https://via.placeholder.com/150", "https://via.placeholder.com/150", "https://via.placeholder.com/150"} - comic.ImagesFormat = "jpg" - - opt := &config.Options{ - OutputFolder: filepath.Dir(os.Args[0]), - CreateDefaultPath: true, - Debug: false, - Logger: logger.NewLogger(false, make(chan string)), - Client: http.NewComicClient(), + opts := newTestOptions(t, server) + + comic := &Comic{ + Name: "baz", + Source: "test-source", + IssueNumber: "7", + Format: CBZ, + ImagesFormat: "png", + Links: buildLinks(server, 2), } - time.Sleep(10 * time.Second) - _, err := comic.DownloadImages(opt) - assert.Nil(t, err) -} + require.NoError(t, comic.MakeComic(opts)) -func TestDownloadImagesJPEGFormat(t *testing.T) { - comic := new(Comic) - - comic.Name = "bar-jpeg" - comic.Source = "fake" - comic.IssueNumber = "example-chapter-1" - comic.ImagesFormat = "jpeg" - comic.Links = []string{"https://via.placeholder.com/150", "https://via.placeholder.com/150", "https://via.placeholder.com/150"} - - opt := &config.Options{ - OutputFolder: filepath.Dir(os.Args[0]), - CreateDefaultPath: true, - Debug: false, - Logger: logger.NewLogger(false, make(chan string)), - Client: http.NewComicClient(), - } - time.Sleep(10 * time.Second) - _, err := comic.DownloadImages(opt) - assert.Nil(t, err) + output := filepath.Join(opts.OutputFolder, "comics", comic.Source, comic.Name, "baz-7.cbz") + require.FileExists(t, output) } -func TestDownloadImagesIMGFormat(t *testing.T) { - comic := new(Comic) - - comic.Name = "bar-img" - comic.Source = "fake" - comic.IssueNumber = "example-chapter-1" - comic.Links = []string{"https://via.placeholder.com/150", "https://via.placeholder.com/150", "https://via.placeholder.com/150"} - comic.ImagesFormat = "img" - - opt := &config.Options{ - OutputFolder: filepath.Dir(os.Args[0]), - CreateDefaultPath: true, - Debug: false, - Logger: logger.NewLogger(false, make(chan string)), - Client: http.NewComicClient(), +func readFile(path string) ([]byte, error) { + file, err := os.Open(path) + if err != nil { + return nil, err } - time.Sleep(10 * time.Second) - _, err := comic.DownloadImages(opt) + defer file.Close() - assert.Nil(t, err) + return io.ReadAll(file) } diff --git a/pkg/detector/detector.go b/pkg/detector/detector.go index 5a966dce..3602c9a5 100644 --- a/pkg/detector/detector.go +++ b/pkg/detector/detector.go @@ -16,7 +16,7 @@ var SupportedSites = map[string]map[string]bool{ "manganato": {"isDisabled": false}, "mangatown": {"isDisabled": false}, "readallcomics": {"isDisabled": false}, - "readcomiconline": {"isDisabled": false}, + "readcomicsonline": {"isDisabled": false}, } // DetectComic will look for the url source to check if a source is supported. diff --git a/pkg/http/client.go b/pkg/http/client.go index acb8e009..87c47d28 100644 --- a/pkg/http/client.go +++ b/pkg/http/client.go @@ -1,40 +1,248 @@ package http import ( + "context" + "errors" + "fmt" "net/http" "strings" + "sync/atomic" + "time" ) -// ComicClient is the custom client. +const ( + defaultTimeout = 15 * time.Second + defaultRetryCount = 2 + defaultRetryWait = 500 * time.Millisecond + defaultUserAgent = "comics-downloader-client" +) + +// RateLimiter exposes a minimal interface for throttling outgoing requests. +type RateLimiter interface { + Wait(context.Context) error +} + +// Option modifies ComicClient behaviour. +type Option func(*ComicClient) + +// WithHTTPClient allows supplying a custom http.Client. +func WithHTTPClient(client *http.Client) Option { + return func(cc *ComicClient) { + if client != nil { + cc.client = client + } + } +} + +// WithRetry configures retry attempts and wait duration between retries. +func WithRetry(count int, wait time.Duration) Option { + return func(cc *ComicClient) { + if count >= 0 { + cc.retryCount = count + } + if wait >= 0 { + cc.retryWait = wait + } + } +} + +// WithRateLimiter sets a rate limiter to gate outbound requests. +func WithRateLimiter(limiter RateLimiter) Option { + return func(cc *ComicClient) { + cc.rateLimiter = limiter + } +} + +// WithUserAgent overrides the default user-agent header with a single value. +func WithUserAgent(agent string) Option { + return WithUserAgents([]string{agent}) +} + +// WithUserAgents configures a list of User-Agent values to rotate through per request. +func WithUserAgents(agents []string) Option { + return func(cc *ComicClient) { + filtered := filterNonEmpty(agents) + if len(filtered) > 0 { + cc.userAgents = filtered + } + } +} + +// WithHeaders adds static headers to every request. +func WithHeaders(headers map[string]string) Option { + return func(cc *ComicClient) { + if len(headers) == 0 { + return + } + if cc.headers == nil { + cc.headers = make(map[string]string) + } + for key, value := range headers { + if strings.TrimSpace(key) == "" || strings.TrimSpace(value) == "" { + continue + } + cc.headers[key] = value + } + } +} + +// ComicClient is the custom HTTP helper used across the downloader. type ComicClient struct { - Client *http.Client + client *http.Client + retryCount int + retryWait time.Duration + rateLimiter RateLimiter + userAgents []string + headers map[string]string + uaCounter uint32 +} + +// NewComicClient returns a ComicClient instance with sane defaults. +func NewComicClient(options ...Option) *ComicClient { + cc := &ComicClient{ + client: &http.Client{ + Timeout: defaultTimeout, + }, + retryCount: defaultRetryCount, + retryWait: defaultRetryWait, + userAgents: []string{defaultUserAgent}, + headers: make(map[string]string), + } + + for _, opt := range options { + opt(cc) + } + + if cc.client.Timeout == 0 { + cc.client.Timeout = defaultTimeout + } + + if len(cc.userAgents) == 0 { + cc.userAgents = []string{defaultUserAgent} + } + + return cc } -// NewComicClient returns a ComicClient instance. -func NewComicClient() *ComicClient { - return &ComicClient{ - Client: &http.Client{}, +// HTTPClient exposes the underlying http.Client instance. +func (c *ComicClient) HTTPClient() *http.Client { + if c == nil { + return nil } + return c.client } -// PrepareRequest setup a `GET` request with customs headers. +// PrepareRequest setup a `GET` request with custom headers. func (c *ComicClient) PrepareRequest(link, hostname string) (*http.Request, error) { - req, err := http.NewRequest("GET", link, nil) + req, err := http.NewRequest(http.MethodGet, link, nil) + if err != nil { + return nil, err + } if strings.Contains(hostname, "manganato") || strings.Contains(hostname, "mangakakalot") { - req.Header.Add("Referer", link) + req.Header.Set("Referer", link) } - return req, err + if c != nil { + req.Header.Set("User-Agent", c.selectUserAgent()) + for key, value := range c.headers { + if strings.EqualFold(key, "user-agent") { + continue + } + req.Header.Set(key, value) + } + } + + return req, nil } -// GET Performs a Get request.. +// Do executes an HTTP request applying retry, timeout, and rate limiting policies. +func (c *ComicClient) Do(req *http.Request) (*http.Response, error) { + if c == nil { + return nil, errors.New("comic client is nil") + } + if req == nil { + return nil, errors.New("request is nil") + } + + attempts := c.retryCount + 1 + if attempts < 1 { + attempts = 1 + } + + var lastErr error + for attempt := 0; attempt < attempts; attempt++ { + if attempt > 0 && c.retryWait > 0 { + select { + case <-time.After(c.retryWait): + case <-req.Context().Done(): + return nil, req.Context().Err() + } + } + + if err := c.wait(req.Context()); err != nil { + return nil, err + } + + currentReq := req + if attempt > 0 { + currentReq = req.Clone(req.Context()) + } + + resp, err := c.client.Do(currentReq) + if err != nil { + lastErr = err + continue + } + + if resp.StatusCode >= 500 { + lastErr = fmt.Errorf("server error: %d", resp.StatusCode) + if closeErr := resp.Body.Close(); closeErr != nil { + lastErr = fmt.Errorf("%w; close error: %v", lastErr, closeErr) + } + continue + } + + return resp, nil + } + + return nil, lastErr +} + +// GET performs a GET request applying the configured policies. func (c *ComicClient) Get(link, hostname string) (*http.Response, error) { request, err := c.PrepareRequest(link, hostname) - if err != nil { return nil, err } - return c.Client.Do(request) + return c.Do(request) +} + +func (c *ComicClient) wait(ctx context.Context) error { + if c.rateLimiter == nil { + return nil + } + return c.rateLimiter.Wait(ctx) +} + +func (c *ComicClient) selectUserAgent() string { + if len(c.userAgents) == 0 { + return defaultUserAgent + } + if len(c.userAgents) == 1 { + return c.userAgents[0] + } + index := int(atomic.AddUint32(&c.uaCounter, 1)-1) % len(c.userAgents) + return c.userAgents[index] +} + +func filterNonEmpty(values []string) []string { + var out []string + for _, value := range values { + if trimmed := strings.TrimSpace(value); trimmed != "" { + out = append(out, trimmed) + } + } + return out } diff --git a/pkg/http/client_test.go b/pkg/http/client_test.go index 2572c63e..067fd784 100644 --- a/pkg/http/client_test.go +++ b/pkg/http/client_test.go @@ -1,26 +1,123 @@ package http import ( + "context" + "net/http" + "net/http/httptest" + "sync/atomic" "testing" + "time" - "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) +type stubLimiter struct { + count int32 +} + +func (s *stubLimiter) Wait(ctx context.Context) error { + atomic.AddInt32(&s.count, 1) + return nil +} + func TestPrepareRequestMangakakalot(t *testing.T) { cc := NewComicClient() link := "http://mangakakalot.com" source := "mangakakalot.com" req, err := cc.PrepareRequest(link, source) - assert.Equal(t, req.Header["Referer"], []string{link}) - assert.Nil(t, err) + require.NoError(t, err) + require.Equal(t, link, req.Header.Get("Referer")) + require.Equal(t, defaultUserAgent, req.Header.Get("User-Agent")) } -func TestPrepareRequest(t *testing.T) { +func TestPrepareRequestGenericHost(t *testing.T) { cc := NewComicClient() link := "http://foo.com" req, err := cc.PrepareRequest(link, "foo") - assert.Equal(t, len(req.Header["Referer"]), 0) - assert.Nil(t, err) + require.NoError(t, err) + require.Empty(t, req.Header.Values("Referer")) + require.Equal(t, defaultUserAgent, req.Header.Get("User-Agent")) +} + +func TestGetRetriesOnServerError(t *testing.T) { + var hits int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + call := atomic.AddInt32(&hits, 1) + if call == 1 { + w.WriteHeader(http.StatusInternalServerError) + return + } + w.WriteHeader(http.StatusOK) + })) + defer server.Close() + + client := NewComicClient( + WithHTTPClient(server.Client()), + WithRetry(1, 0), + ) + + resp, err := client.Get(server.URL, "example.com") + require.NoError(t, err) + require.NotNil(t, resp) + require.Equal(t, int32(2), atomic.LoadInt32(&hits)) +} + +func TestRateLimiterInvoked(t *testing.T) { + limiter := &stubLimiter{} + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + })) + defer server.Close() + + client := NewComicClient( + WithHTTPClient(server.Client()), + WithRetry(0, 0), + WithRateLimiter(limiter), + ) + + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + + req, err := client.PrepareRequest(server.URL, "example.com") + require.NoError(t, err) + + req = req.WithContext(ctx) + _, err = client.Do(req) + require.NoError(t, err) + require.Equal(t, int32(1), atomic.LoadInt32(&limiter.count)) +} + +func TestUserAgentRotation(t *testing.T) { + agents := []string{"UA-1", "UA-2"} + // we need to ensure rotation occurs across calls + client := NewComicClient(WithUserAgents(agents)) + + req1, err := client.PrepareRequest("http://example.com", "example.com") + require.NoError(t, err) + req2, err := client.PrepareRequest("http://example.com", "example.com") + require.NoError(t, err) + + if req1.Header.Get("User-Agent") == req2.Header.Get("User-Agent") { + t.Fatalf("expected rotating user agents, got identical headers %q", req1.Header.Get("User-Agent")) + } + + req3, err := client.PrepareRequest("http://example.com", "example.com") + require.NoError(t, err) + + require.Equal(t, req1.Header.Get("User-Agent"), req3.Header.Get("User-Agent"), "rotation should loop back to first entry") +} + +func TestAdditionalHeadersApplied(t *testing.T) { + client := NewComicClient(WithHeaders(map[string]string{ + "Cookie": "cf_clearance=abc", + "X-Custom-Id": "123", + })) + + req, err := client.PrepareRequest("http://example.com", "example.com") + require.NoError(t, err) + + require.Equal(t, "cf_clearance=abc", req.Header.Get("Cookie")) + require.Equal(t, "123", req.Header.Get("X-Custom-Id")) } diff --git a/pkg/sites/comicextra.go b/pkg/sites/comicextra.go index 07db0ac4..a31f8228 100644 --- a/pkg/sites/comicextra.go +++ b/pkg/sites/comicextra.go @@ -36,9 +36,9 @@ func (c *Comicextra) retrieveImageLinks(comic *core.Comic) ([]string, error) { match := re.FindAllStringSubmatch(response, -1) for i := range match { - url := match[i][1] - if util.IsURLValid(url) { - links = append(links, url) + link := deobfuscateURL(match[i][1]) + if util.IsURLValid(link) { + links = append(links, link) } } diff --git a/pkg/sites/comicextra_deobfuscate_test.go b/pkg/sites/comicextra_deobfuscate_test.go new file mode 100644 index 00000000..d1a86ff7 --- /dev/null +++ b/pkg/sites/comicextra_deobfuscate_test.go @@ -0,0 +1,21 @@ +package sites + +import "testing" + +func TestDeobfuscateURL(t *testing.T) { + cases := []struct { + input string + expected string + }{ + {`https:\/\/foo.com\/img%3Fid%3D1`, "https://foo.com/img?id=1"}, + {"https://foo.com/img=2&token%3Dabc%252F123", "https://foo.com/img=2&token=abc%2F123"}, + {"", ""}, + } + + for _, tc := range cases { + actual := deobfuscateURL(tc.input) + if actual != tc.expected { + t.Fatalf("deobfuscateURL(%q) = %q, expected %q", tc.input, actual, tc.expected) + } + } +} diff --git a/pkg/sites/comicextra_test.go b/pkg/sites/comicextra_test.go index 9256a244..16feda9a 100644 --- a/pkg/sites/comicextra_test.go +++ b/pkg/sites/comicextra_test.go @@ -1,131 +1,113 @@ package sites import ( + "net/http" + "net/http/httptest" "testing" "github.com/Girbons/comics-downloader/internal/logger" "github.com/Girbons/comics-downloader/pkg/config" "github.com/Girbons/comics-downloader/pkg/core" - "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) -func TestComicExtraSetup(t *testing.T) { - comic := new(core.Comic) - comic.URLSource = "https://comicextra.me/batman-unseen/issue-5/full" +const ( + comicExtraIssueFullPath = "/batman-unseen/issue-5/full" + comicExtraIssueSimple = "/batman-unseen/issue-5" + comicExtraLastIssuePath = "/batman-unseen/issue-4/full" + comicExtraListPath = "/comic/batman-unseen" +) - opt := - &config.Options{ - URL: "https://comicextra.me/batman-unseen/issue-5/full", - All: false, - Last: false, - Debug: false, - Logger: logger.NewLogger(false, make(chan string)), +func newComicExtraServer() *httptest.Server { + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + base := "http://" + r.Host + switch r.URL.Path { + case comicExtraIssueFullPath, comicExtraIssueSimple, "/batman-unseen/issue-4": + html := ` + + + + + + ` + _, _ = w.Write([]byte(html)) + case comicExtraLastIssuePath: + html := ` + + + + + ` + _, _ = w.Write([]byte(html)) + case comicExtraListPath: + html := ` + + +
+ + +
+ + ` + _, _ = w.Write([]byte(html)) + default: + http.NotFound(w, r) } - - comicextra := NewComicextra(opt) - err := comicextra.Initialize(comic) - - assert.Nil(t, err) - assert.Equal(t, 23, len(comic.Links)) + })) } -func TestComicExtraGetInfo(t *testing.T) { - opt := - &config.Options{ - URL: "https://comicextra.me/batman-unseen/issue-5/full", - All: false, - Last: false, - Debug: false, - Logger: logger.NewLogger(false, make(chan string)), - } +func TestComicExtraScraper(t *testing.T) { + server := newComicExtraServer() + defer server.Close() - comicextra := NewComicextra(opt) - name, issueNumber := comicextra.GetInfo("https://comicextra.me/batman-unseen/issue-5/full") + opts := &config.Options{ + URL: server.URL + comicExtraIssueFullPath, + Logger: logger.NewLogger(false, nil), + } - assert.Equal(t, "batman-unseen", name) - assert.Equal(t, "issue-5", issueNumber) -} - -func TestComicextraRetrieveIssueLinks(t *testing.T) { - opt := - &config.Options{ - URL: "https://comicextra.me/batman-unseen/issue-5/full", - All: false, - Last: false, - Debug: false, - Logger: logger.NewLogger(false, make(chan string)), - } + comicextra := NewComicextra(opts) - comicextra := NewComicextra(opt) - issues, err := comicextra.RetrieveIssueLinks() - - assert.Nil(t, err) - assert.Equal(t, 1, len(issues)) + comic := &core.Comic{URLSource: server.URL + comicExtraIssueFullPath} + require.NoError(t, comicextra.Initialize(comic)) + require.Equal(t, []string{ + "https://cdn.example.com/batman?page=1", + "https://cdn.example.com/batman?page=2", + }, comic.Links) } -func TestComicextraRetrieveIssueLinksURLWithPage(t *testing.T) { - opt := - &config.Options{ - URL: "https://comicextra.me/batman-unseen/full", - All: false, - Last: false, - Debug: false, - Logger: logger.NewLogger(false, make(chan string)), - } - - comicextra := NewComicextra(opt) - issues, err := comicextra.RetrieveIssueLinks() +func TestComicExtraRetrieveIssueLinksAll(t *testing.T) { + server := newComicExtraServer() + defer server.Close() - assert.Nil(t, err) - assert.Equal(t, 1, len(issues)) -} + opts := &config.Options{ + URL: server.URL + comicExtraListPath, + All: true, + Logger: logger.NewLogger(false, nil), + } -func TestComicextraRetrieveIssueLinksInASinglePage(t *testing.T) { - opt := - &config.Options{ - URL: "https://comicextra.me/batman-unseen/issue-4/full", - All: false, - Last: false, - Debug: false, - Logger: logger.NewLogger(false, make(chan string)), - } - - comicextra := NewComicextra(opt) - issues, err := comicextra.RetrieveIssueLinks() - - assert.Nil(t, err) - assert.Equal(t, 1, len(issues)) -} - -func TestComicextraRetrieveIssueLinksLastChapter(t *testing.T) { - opt := - &config.Options{ - URL: "https://comicextra.me/batman-unseen/issue-4/full", - All: false, - Last: true, - Debug: false, - Logger: logger.NewLogger(false, make(chan string)), - } - - comicextra := NewComicextra(opt) + comicextra := NewComicextra(opts) issues, err := comicextra.RetrieveIssueLinks() - - assert.Nil(t, err) - assert.Equal(t, 1, len(issues)) + require.NoError(t, err) + require.Equal(t, []string{ + comicExtraIssueFullPath, + "/batman-unseen/issue-4/full", + }, issues) } -func TestComicExtraRetrieveLastIssueLink(t *testing.T) { - comicextra := new(Comicextra) - issue, err := comicextra.retrieveLastIssue("https://comicextra.me/batman-unseen/issue-1/full") +func TestComicExtraRetrieveLastIssue(t *testing.T) { + server := newComicExtraServer() + defer server.Close() - assert.Nil(t, err) - assert.Equal(t, "https://comicextra.me/batman-unseen/issue-5/full", issue) -} - -func TestComicExtraRetrieveLastIssueLinkNotDetail(t *testing.T) { - comicextra := new(Comicextra) - issue, err := comicextra.retrieveLastIssue("https://comicextra.me/batman-unseen/issue-1/full") + opts := &config.Options{ + URL: server.URL + comicExtraLastIssuePath, + Last: true, + Logger: logger.NewLogger(false, nil), + } - assert.Nil(t, err) - assert.Equal(t, "https://comicextra.me/batman-unseen/issue-5/full", issue) + comicextra := NewComicextra(opts) + issues, err := comicextra.RetrieveIssueLinks() + require.NoError(t, err) + require.Equal(t, []string{server.URL + comicExtraIssueFullPath}, issues) } diff --git a/pkg/sites/deobfuscate.go b/pkg/sites/deobfuscate.go new file mode 100644 index 00000000..1d3580e4 --- /dev/null +++ b/pkg/sites/deobfuscate.go @@ -0,0 +1,24 @@ +package sites + +import ( + "html" + "net/url" + "strings" +) + +func deobfuscateURL(raw string) string { + if raw == "" { + return "" + } + + cleaned := strings.ReplaceAll(raw, `\/`, "/") + cleaned = strings.ReplaceAll(cleaned, `\\`, `\`) + + decoded := html.UnescapeString(cleaned) + + if result, err := url.PathUnescape(decoded); err == nil { + decoded = result + } + + return decoded +} diff --git a/pkg/sites/http_helpers.go b/pkg/sites/http_helpers.go new file mode 100644 index 00000000..0b4d202b --- /dev/null +++ b/pkg/sites/http_helpers.go @@ -0,0 +1,86 @@ +package sites + +import ( + "context" + "encoding/json" + "fmt" + "io" + "log" + "net/http" + urlpkg "net/url" + + httpclient "github.com/Girbons/comics-downloader/pkg/http" +) + +func defaultClient(client *httpclient.ComicClient) *httpclient.ComicClient { + if client != nil { + return client + } + return httpclient.NewComicClient() +} + +func hostFromURL(link string) string { + parsed, err := urlpkg.Parse(link) + if err != nil { + return "" + } + return parsed.Host +} + +func buildRequest(ctx context.Context, client *httpclient.ComicClient, link string) (*http.Request, error) { + req, err := client.PrepareRequest(link, hostFromURL(link)) + if err != nil { + return nil, err + } + if ctx != nil { + req = req.WithContext(ctx) + } + return req, nil +} + +func fetchJSON(ctx context.Context, client *httpclient.ComicClient, link string, target interface{}) error { + if target == nil { + return fmt.Errorf("target cannot be nil") + } + + data, err := fetchBytes(ctx, client, link) + if err != nil { + return err + } + if len(data) == 0 { + return fmt.Errorf("empty response for %s", link) + } + return json.Unmarshal(data, target) +} + +func fetchBytes(ctx context.Context, client *httpclient.ComicClient, link string) ([]byte, error) { + cc := defaultClient(client) + if ctx == nil { + ctx = context.Background() + } + + req, err := buildRequest(ctx, cc, link) + if err != nil { + return nil, err + } + + resp, err := cc.Do(req) + if err != nil { + return nil, err + } + defer func() { + if closeErr := resp.Body.Close(); closeErr != nil { + log.Printf("sites: failed to close response body for %s: %v", link, closeErr) + } + }() + + if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices { + return nil, fmt.Errorf("unexpected status code %d for %s", resp.StatusCode, link) + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + return body, nil +} diff --git a/pkg/sites/loader.go b/pkg/sites/loader.go index 4125853b..e6b524bd 100644 --- a/pkg/sites/loader.go +++ b/pkg/sites/loader.go @@ -1,7 +1,6 @@ package sites import ( - "errors" "fmt" "regexp" "strconv" @@ -18,7 +17,7 @@ func initializeCollection(issues []string, options *config.Options, base BaseSit var err error if len(issues) == 0 { - return collection, errors.New("No issues found") + return collection, fmt.Errorf("no issues found for URL %q; ensure it points to a specific comic or chapter page", options.URL) } var startRange, endRange float64 @@ -43,7 +42,10 @@ func initializeCollection(issues []string, options *config.Options, base BaseSit continue } - dir, _ := util.PathSetup(options.CreateDefaultPath, options.OutputFolder, options.Source, name) + dir, pathErr := util.PathSetup(options.CreateDefaultPath, options.OutputFolder, options.Source, name) + if pathErr != nil { + return collection, pathErr + } fileName := util.GetPathToFile(dir, name, issueNumber, options.Format, options.IssueNumberNameOnly) if util.DirectoryOrFileDoesNotExist(fileName) || options.ImagesOnly { @@ -95,8 +97,8 @@ func LoadComicFromSource(options *config.Options) ([]*core.Comic, error) { ) switch { - case strings.Contains(options.Source, "readcomiconline"): - base = NewReadComiconline(options) + case strings.Contains(options.Source, "readcomicsonline.ru"): + base = NewReadComicsOnline(options) case strings.Contains(options.Source, "comicextra"): base = NewComicextra(options) case strings.Contains(options.Source, "mangareader"): @@ -116,10 +118,18 @@ func LoadComicFromSource(options *config.Options) ([]*core.Comic, error) { return collection, err } + if options.Logger != nil && options.Debug { + options.Logger.Debugf("sites: retrieving issues for %s", options.URL) + } + issues, err = base.RetrieveIssueLinks() if err != nil { return collection, err } + if options.Logger != nil && options.Debug { + options.Logger.Debugf("sites: %d issue(s) discovered for %s", len(issues), options.URL) + } + return initializeCollection(issues, options, base) } diff --git a/pkg/sites/loader_test.go b/pkg/sites/loader_test.go index 8ee3f9f3..6204740d 100644 --- a/pkg/sites/loader_test.go +++ b/pkg/sites/loader_test.go @@ -1,200 +1,84 @@ package sites import ( - "fmt" - "os" - "path/filepath" + "errors" "testing" "github.com/Girbons/comics-downloader/pkg/config" - "github.com/stretchr/testify/assert" + "github.com/Girbons/comics-downloader/pkg/core" + "github.com/stretchr/testify/require" ) -func TestSiteLoaderMangatown(t *testing.T) { - url := "https://www.mangatown.com/manga/naruto/v63/c693/" - outputFolder := filepath.Dir(os.Args[0]) - - options := &config.Options{ - All: false, - Last: false, - ImagesOnly: false, - Source: "www.mangatown.com", - URL: url, - Format: "pdf", - ImagesFormat: "png", - OutputFolder: outputFolder, - } - - collection, err := LoadComicFromSource(options) - - assert.Nil(t, err) - assert.Equal(t, len(collection), 1) - - comic := collection[0] - - assert.Equal(t, "www.mangatown.com", comic.Source) - assert.Equal(t, url, comic.URLSource) - assert.Equal(t, "naruto", comic.Name) - assert.Equal(t, "c693", comic.IssueNumber) - assert.Equal(t, 20, len(comic.Links)) +type stubSite struct { + issues []string + comics map[string]*core.Comic } -func TestCustomComicName(t *testing.T) { - url := "https://www.mangatown.com/manga/naruto/v63/c693/" - outputFolder := filepath.Dir(os.Args[0]) - - options := &config.Options{ - All: false, - Last: false, - ImagesOnly: false, - Source: "www.mangatown.com", - URL: url, - Format: "pdf", - ImagesFormat: "png", - CustomComicName: "Naruto", - OutputFolder: outputFolder, +func (s *stubSite) Initialize(comic *core.Comic) error { + if stub, ok := s.comics[comic.URLSource]; ok { + *comic = *stub + return nil } - - collection, err := LoadComicFromSource(options) - - assert.Nil(t, err) - assert.Equal(t, len(collection), 1) - - comic := collection[0] - - assert.Equal(t, "www.mangatown.com", comic.Source) - assert.Equal(t, url, comic.URLSource) - assert.Equal(t, "Naruto", comic.Name) - assert.Equal(t, "c693", comic.IssueNumber) - assert.Equal(t, 20, len(comic.Links)) + return errors.New("missing comic") } -//func TestSiteLoaderMangareader(t *testing.T) { -//url := "https://www.mangareader.net/naruto/700" -//outputFolder := filepath.Dir(os.Args[0]) - -//options := &config.Options{ -//All: false, -//Last: false, -//ImagesOnly: false, -//Source: "www.mangareader.net", -//Url: url, -//Format: "pdf", -//ImagesFormat: "png", -//OutputFolder: outputFolder, -//} - -//collection, err := LoadComicFromSource(options) - -//assert.Nil(t, err) -//assert.Equal(t, len(collection), 1) - -//comic := collection[0] - -//assert.Equal(t, "www.mangareader.net", comic.Source) -//assert.Equal(t, url, comic.URLSource) -//assert.Equal(t, "naruto", comic.Name) -//assert.Equal(t, "700", comic.IssueNumber) -//assert.Equal(t, 23, len(comic.Links)) -//} - -func TestSiteLoaderComicExtra(t *testing.T) { - url := "https://comicextra.me/batman-unseen/issue-5/full" - outputFolder := filepath.Dir(os.Args[0]) - options := &config.Options{ - All: false, - Last: false, - ImagesOnly: false, - Source: "comicextra.net", - URL: url, - Format: "pdf", - ImagesFormat: "png", - OutputFolder: outputFolder, +func (s *stubSite) GetInfo(url string) (string, string) { + if stub, ok := s.comics[url]; ok { + return stub.Name, stub.IssueNumber } - collection, err := LoadComicFromSource(options) - - assert.Nil(t, err) - assert.Equal(t, 1, len(collection)) - - comic := collection[0] - - assert.Equal(t, "comicextra.net", comic.Source) - assert.Equal(t, url, comic.URLSource) - assert.Equal(t, "batman-unseen", comic.Name) - assert.Equal(t, "issue-5", comic.IssueNumber) - assert.Equal(t, 23, len(comic.Links)) + return "", "" } -func TestLoaderUnknownSource(t *testing.T) { - url := "http://example.com" - outputFolder := filepath.Dir(os.Args[0]) +func (s *stubSite) RetrieveIssueLinks() ([]string, error) { + return s.issues, nil +} +func TestInitializeCollectionFiltersIssues(t *testing.T) { options := &config.Options{ - All: false, - Last: false, - ImagesOnly: false, - Source: "example.com", - URL: url, + Source: "test-source", Format: "pdf", ImagesFormat: "png", - OutputFolder: outputFolder, + IssuesRange: "1-2", + All: true, } - collection, err := LoadComicFromSource(options) - - if assert.NotNil(t, err) { - assert.Equal(t, fmt.Errorf("source unknown"), err) + site := &stubSite{ + issues: []string{"url-1", "url-2", "url-3"}, + comics: map[string]*core.Comic{ + "url-1": {Name: "series", IssueNumber: "issue-1", URLSource: "url-1"}, + "url-2": {Name: "series", IssueNumber: "issue-2", URLSource: "url-2"}, + "url-3": {Name: "series", IssueNumber: "issue-3", URLSource: "url-3"}, + }, } - assert.Equal(t, len(collection), 0) + + collection, err := initializeCollection(site.issues, options, site) + require.NoError(t, err) + require.Len(t, collection, 2) + require.Equal(t, "issue-1", collection[0].IssueNumber) + require.Equal(t, "issue-2", collection[1].IssueNumber) } -func TestIssuesRange(t *testing.T) { - url := "https://comicextra.net/batman-unseen/issue-5/full" - outputFolder := filepath.Dir(os.Args[0]) - options := &config.Options{ - All: true, - Last: false, - ImagesOnly: false, - Source: "comicextra.net", - URL: url, - Format: "pdf", - ImagesFormat: "png", - OutputFolder: outputFolder, - IssuesRange: "1-3", - } +func TestLoadComicFromSourceUnknown(t *testing.T) { + options := &config.Options{Source: "unknown"} collection, err := LoadComicFromSource(options) - - assert.Nil(t, err) - assert.Equal(t, len(collection), 3) - - issues := make([]string, 0, len(collection)) - for _, c := range collection { - issues = append(issues, c.IssueNumber) - } - - assert.Contains(t, issues, "issue-1") - assert.Contains(t, issues, "issue-2") - assert.Contains(t, issues, "issue-3") + require.Error(t, err) + require.Empty(t, collection) } -func TestFloatIssuesRange(t *testing.T) { - tt := []struct { - input string - start float64 - end float64 - returnValue bool +func TestNotInIssuesRange(t *testing.T) { + testCases := []struct { + issue string + start float64 + end float64 + skip bool }{ - {"1", 1, 1, false}, - {"19", 20, 21, true}, - {"20", 20, 21, false}, - {"20.5", 20, 21, false}, - {"21", 20, 21, false}, - {"22", 20, 21, true}, + {"1", 1, 2, false}, + {"3", 1, 2, true}, + {"2.5", 2, 3, false}, + {"abc", 1, 2, true}, } - for _, tc := range tt { - t.Run(tc.input, func(t *testing.T) { - assert.Equal(t, notInIssuesRange(tc.input, tc.start, tc.end), tc.returnValue) - }) + for _, tc := range testCases { + require.Equal(t, tc.skip, notInIssuesRange(tc.issue, tc.start, tc.end)) } } diff --git a/pkg/sites/mangadex.go b/pkg/sites/mangadex.go index a550c812..e7e3b942 100644 --- a/pkg/sites/mangadex.go +++ b/pkg/sites/mangadex.go @@ -1,39 +1,68 @@ package sites import ( + "context" "encoding/json" "errors" "fmt" - "net/http" + "net/url" "strings" + "time" "github.com/Girbons/comics-downloader/pkg/config" "github.com/Girbons/comics-downloader/pkg/core" + httpclient "github.com/Girbons/comics-downloader/pkg/http" "github.com/Girbons/comics-downloader/pkg/util" ) +const ( + mangadexAPIBase = "https://api.mangadex.org" + mangadexChapterBase = "https://mangadex.org/chapter" + mangadexUploadsBase = "https://uploads.mangadex.org/data" + mangadexRequestTimeout = 8 * time.Second +) + // Mangadex represents a mangadex instance. type Mangadex struct { - country string - baseURL string - options *config.Options + country string + options *config.Options + client *httpclient.ComicClient + apiBase string + chapterBase string + uploadsBase string } -// NewMangadex returns a Mangadex instance +// NewMangadex returns a Mangadex instance. func NewMangadex(options *config.Options) *Mangadex { + client := options.Client + if client == nil { + client = httpclient.NewComicClient() + options.Client = client + } + return &Mangadex{ - country: strings.ToLower(options.Country), - options: options, + country: strings.ToLower(options.Country), + options: options, + client: client, + apiBase: mangadexAPIBase, + chapterBase: mangadexChapterBase, + uploadsBase: mangadexUploadsBase, } } -func (m *Mangadex) getManga(mangaID string) (title string, err error) { - url := fmt.Sprintf("https://api.mangadex.org/manga/%s", mangaID) - res, err := http.Get(url) - if err != nil { - return "", err - } - defer res.Body.Close() +func (m *Mangadex) requestContext() (context.Context, context.CancelFunc) { + return context.WithTimeout(context.Background(), mangadexRequestTimeout) +} + +func joinURL(base, suffix string) string { + return strings.TrimRight(base, "/") + "/" + strings.TrimLeft(suffix, "/") +} + +func (m *Mangadex) getManga(mangaID string) (string, error) { + ctx, cancel := m.requestContext() + defer cancel() + + endpoint := joinURL(m.apiBase, fmt.Sprintf("/manga/%s", mangaID)) var mangaRes struct { Result string `json:"result"` Data struct { @@ -42,77 +71,87 @@ func (m *Mangadex) getManga(mangaID string) (title string, err error) { } `json:"attributes"` } `json:"data"` } - if err := json.NewDecoder(res.Body).Decode(&mangaRes); err != nil { + + if err := fetchJSON(ctx, m.client, endpoint, &mangaRes); err != nil { return "", err } - if mangaRes.Result != "ok" { - return "", fmt.Errorf("Unexpected response") + if strings.ToLower(mangaRes.Result) != "ok" { + return "", fmt.Errorf("unexpected response") } + for lang, t := range mangaRes.Data.Attributes.Titles { - title = t - if m.country == "" || m.country == lang { - break + if m.country == "" || m.country == strings.ToLower(lang) { + return t, nil } } - return title, nil + + // Fallback to any available title. + for _, t := range mangaRes.Data.Attributes.Titles { + return t, nil + } + + return "", fmt.Errorf("no title found for manga %s", mangaID) } -// Get a list of chapter IDs of a manga. +// getChapters fetches chapter URLs for the given manga. func (m *Mangadex) getChapters(mangaID string) ([]string, error) { - url := fmt.Sprintf("https://api.mangadex.org/manga/%s/aggregate?", mangaID) + ctx, cancel := m.requestContext() + defer cancel() + + endpoint := joinURL(m.apiBase, fmt.Sprintf("/manga/%s/aggregate", mangaID)) if m.country != "" { - url += fmt.Sprintf("&translatedLanguage[]=%s", m.country) + q := url.Values{} + q.Add("translatedLanguage[]", m.country) + endpoint += "?" + q.Encode() } - res, err := http.Get(url) + + body, err := fetchBytes(ctx, m.client, endpoint) if err != nil { return nil, err } - defer res.Body.Close() + if len(body) == 0 || body[0] == '[' { + return []string{}, nil + } + var chaptersRes struct { Result string `json:"result"` Volumes map[string]struct { - Name string `json:"volume"` Chapters map[string]struct { ID string `json:"id"` Name string `json:"chapter"` } `json:"chapters"` } `json:"volumes"` } - if err := json.NewDecoder(res.Body).Decode(&chaptersRes); err != nil { - // This is not ideal. MangaDex returns an empty array (`[]`) when no volume - // exists and a `map[string]interface{}` otherwise. - return []string{}, nil + + if err := json.Unmarshal(body, &chaptersRes); err != nil { + return nil, err } - if chaptersRes.Result != "ok" { - return nil, fmt.Errorf("Unexpected response") + if strings.ToLower(chaptersRes.Result) != "ok" { + return nil, fmt.Errorf("unexpected response") } + var ids []string for _, v := range chaptersRes.Volumes { for _, c := range v.Chapters { - url := fmt.Sprintf("https://mangadex.org/chapter/%s", c.ID) - ids = append(ids, url) + ids = append(ids, joinURL(m.chapterBase, c.ID)) } } return ids, nil } -// Get a list of a chapters images. +// getChapter retrieves metadata and image links for a single chapter. func (m *Mangadex) getChapter(chapterID string) (mangaID, volume, chapter, title string, images []string, err error) { - url := fmt.Sprintf("https://api.mangadex.org/chapter/%s", chapterID) - res, err := http.Get(url) - if err != nil { - return "", "", "", "", nil, err - } - defer res.Body.Close() + ctx, cancel := m.requestContext() + defer cancel() + + endpoint := joinURL(m.apiBase, fmt.Sprintf("/chapter/%s", chapterID)) var chapterRes struct { Result string `json:"result"` Data struct { Attributes struct { - Volume string `json:"volume"` - Chapter string `json:"chapter"` - Title string `json:"title"` - Hash string `json:"hash"` - Data []string `json:"data"` + Volume string `json:"volume"` + Chapter string `json:"chapter"` + Title string `json:"title"` } `json:"attributes"` Relationships []struct { ID string `json:"id"` @@ -121,13 +160,14 @@ func (m *Mangadex) getChapter(chapterID string) (mangaID, volume, chapter, title } `json:"data"` } - if err := json.NewDecoder(res.Body).Decode(&chapterRes); err != nil { + if err := fetchJSON(ctx, m.client, endpoint, &chapterRes); err != nil { return "", "", "", "", nil, err } - if chapterRes.Result != "ok" { - return "", "", "", "", nil, fmt.Errorf("Unexpected response") + if strings.ToLower(chapterRes.Result) != "ok" { + return "", "", "", "", nil, fmt.Errorf("unexpected response") } + imagesEndpoint := joinURL(m.apiBase, fmt.Sprintf("/at-home/server/%s", chapterID)) var imagesRes struct { Result string `json:"result"` Chapter struct { @@ -136,20 +176,22 @@ func (m *Mangadex) getChapter(chapterID string) (mangaID, volume, chapter, title } `json:"chapter"` } - res, err = http.Get(fmt.Sprintf("https://api.mangadex.org/at-home/server/%s", chapterID)) - - if err := json.NewDecoder(res.Body).Decode(&imagesRes); err != nil { + if err := fetchJSON(ctx, m.client, imagesEndpoint, &imagesRes); err != nil { return "", "", "", "", nil, err } + if strings.ToLower(imagesRes.Result) != "ok" { + return "", "", "", "", nil, fmt.Errorf("unexpected response") + } for _, file := range imagesRes.Chapter.Data { - imageUrl := fmt.Sprintf("https://uploads.mangadex.org/data/%s/%s", imagesRes.Chapter.Hash, file) - images = append(images, imageUrl) + imageURL := joinURL(m.uploadsBase, fmt.Sprintf("%s/%s", imagesRes.Chapter.Hash, file)) + images = append(images, imageURL) } - if m.options.Debug { + if m.options.Debug && len(images) > 0 && m.options.Logger != nil { m.options.Logger.Debug(fmt.Sprintf("Image Links found: %s", strings.Join(images, " "))) } + for _, rel := range chapterRes.Data.Relationships { if rel.Type == "manga" { mangaID = rel.ID @@ -177,8 +219,8 @@ func (m *Mangadex) RetrieveIssueLinks() ([]string, error) { } // GetInfo extracts the basic info from the given url. -func (m *Mangadex) GetInfo(url string) (string, string) { - parts := util.TrimAndSplitURL(url) +func (m *Mangadex) GetInfo(urlValue string) (string, string) { + parts := util.TrimAndSplitURL(urlValue) if len(parts) < 5 { return "", "" } @@ -208,10 +250,10 @@ func (m *Mangadex) GetInfo(url string) (string, string) { } } -// Initialize loads links and metadata from mangadex +// Initialize loads links and metadata from mangadex. func (m *Mangadex) Initialize(comic *core.Comic) error { parts := util.TrimAndSplitURL(comic.URLSource) - if len(parts) < 4 { + if len(parts) < 5 { return fmt.Errorf("URL not supported") } _, _, _, _, images, err := m.getChapter(parts[4]) diff --git a/pkg/sites/mangadex_test.go b/pkg/sites/mangadex_test.go index 6c9167ea..92353413 100644 --- a/pkg/sites/mangadex_test.go +++ b/pkg/sites/mangadex_test.go @@ -1,144 +1,124 @@ package sites -//import ( -//"testing" - -//"github.com/Girbons/comics-downloader/internal/logger" -//"github.com/Girbons/comics-downloader/pkg/config" -//"github.com/Girbons/comics-downloader/pkg/core" -//"github.com/stretchr/testify/assert" -//) - -//const testMangadexBase string = "mangadex.org" -//const testMangadexURL string = "https://" + testMangadexBase + "/" - -//func TestMangadexGetInfo(t *testing.T) { -//opt := &config.Options{ -//Url: testMangadexURL + "chapter/155061/1", -//Country: "", -//Source: testMangadexBase, -//Debug: false, -//Logger: logger.NewLogger(false, make(chan string)), -//} -//md := NewMangadex(opt) -//name, issueNumber := md.GetInfo(testMangadexURL + "chapter/155061/1") - -//assert.Equal(t, "Naruto", name) -//assert.Equal(t, "Vol 60 Chapter 575, A Will of Stone", issueNumber) -//} - -//func TestMangadexSetup(t *testing.T) { -//opt := &config.Options{ -//Url: testMangadexURL + "chapter/155061/1", -//Country: "", -//Source: testMangadexBase, -//Debug: false, -//Logger: logger.NewLogger(false, make(chan string)), -//} -//comic := new(core.Comic) -//comic.URLSource = testMangadexURL + "chapter/155061/1" - -//md := NewMangadex(opt) -//err := md.Initialize(comic) - -//assert.Nil(t, err) -//assert.Equal(t, 14, len(comic.Links)) -//} - -//func TestMangadexRetrieveIssueLinks(t *testing.T) { -//opt := &config.Options{ -//Url: testMangadexURL + "chapter/155061/", -//Country: "", -//Source: testMangadexBase, -//Last: false, -//All: false, -//Debug: false, -//Logger: logger.NewLogger(false, make(chan string)), -//} -//md := NewMangadex(opt) -//urls, err := md.RetrieveIssueLinks() -//assert.Nil(t, err) -//assert.Equal(t, 1, len(urls)) -//} - -//func TestMangadexRetrieveIssueLinksAllChapter(t *testing.T) { -//opt := &config.Options{ -//Url: testMangadexURL + "title/5/naruto/", -//Country: "gb", -//Source: testMangadexBase, -//Last: false, -//All: true, -//Debug: false, -//Logger: logger.NewLogger(false, make(chan string)), -//} -//md := NewMangadex(opt) -//urls, err := md.RetrieveIssueLinks() -//assert.Nil(t, err) -//assert.Len(t, urls, 713) -//} - -//func TestMangadexRetrieveIssueLinksLastChapter(t *testing.T) { -//opt := &config.Options{ -//Url: testMangadexURL + "title/5/naruto/", -//Country: "gb", -//Source: testMangadexBase, -//Last: true, -//All: false, -//Debug: false, -//Logger: logger.NewLogger(false, make(chan string)), -//} -//md := NewMangadex(opt) -//urls, err := md.RetrieveIssueLinks() -//assert.Nil(t, err) -//assert.Len(t, urls, 1) -//} - -//func TestMangadexUnsupportedURL(t *testing.T) { -//opt := &config.Options{ -//Url: testMangadexURL, -//Country: "", -//Source: testMangadexBase, -//Last: false, -//All: false, -//Debug: false, -//Logger: logger.NewLogger(false, make(chan string)), -//} -//md := NewMangadex(opt) -//_, err := md.RetrieveIssueLinks() -//assert.EqualError(t, err, "URL not supported") - -//md.options.Url = testMangadexURL + "test/0/" -//_, err = md.RetrieveIssueLinks() -//assert.EqualError(t, err, "URL not supported") -//} - -//func TestMangadexNoManga(t *testing.T) { -//opt := &config.Options{ -//Url: testMangadexURL + "title/0/", -//Country: "", -//Source: testMangadexBase, -//Last: false, -//All: false, -//Debug: false, -//Logger: logger.NewLogger(false, make(chan string)), -//} -//md := NewMangadex(opt) -//_, err := md.RetrieveIssueLinks() -//assert.Error(t, err) -//assert.Contains(t, err.Error(), "could not get manga 0") -//} - -//func TestMangadexNoChapters(t *testing.T) { -//opt := &config.Options{ -//Url: testMangadexURL + "title/5/naruto/", -//Country: "xyz", -//Source: testMangadexBase, -//Last: false, -//All: true, -//Debug: false, -//Logger: logger.NewLogger(false, make(chan string)), -//} -//md := NewMangadex(opt) -//_, err := md.RetrieveIssueLinks() -//assert.EqualError(t, err, "no chapters found") -//} +import ( + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/Girbons/comics-downloader/internal/logger" + "github.com/Girbons/comics-downloader/pkg/config" + "github.com/Girbons/comics-downloader/pkg/core" + httpclient "github.com/Girbons/comics-downloader/pkg/http" + "github.com/stretchr/testify/require" +) + +func setupMangadexServer() *httptest.Server { + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case strings.HasPrefix(r.URL.Path, "/manga/series-1/aggregate"): + w.Header().Set("Content-Type", "application/json") + fmt.Fprint(w, `{ + "result":"ok", + "volumes":{ + "1":{ + "chapters":{ + "1":{"id":"chapter-1","chapter":"1"} + } + } + } + }`) + case strings.HasPrefix(r.URL.Path, "/manga/series-1"): + w.Header().Set("Content-Type", "application/json") + fmt.Fprint(w, `{ + "result":"ok", + "data":{ + "attributes":{ + "title":{"en":"Test Manga","jp":"テスト"} + } + } + }`) + case strings.HasPrefix(r.URL.Path, "/chapter/chapter-1"): + w.Header().Set("Content-Type", "application/json") + fmt.Fprint(w, `{ + "result":"ok", + "data":{ + "attributes":{"volume":"1","chapter":"1","title":"Start"}, + "relationships":[{"id":"series-1","type":"manga"}] + } + }`) + case strings.HasPrefix(r.URL.Path, "/at-home/server/chapter-1"): + w.Header().Set("Content-Type", "application/json") + fmt.Fprint(w, `{ + "result":"ok", + "chapter":{"hash":"HASH","data":["001.png","002.png"]} + }`) + default: + http.NotFound(w, r) + } + })) +} + +func newTestMangadex(t *testing.T) (*Mangadex, func()) { + t.Helper() + + server := setupMangadexServer() + + client := httpclient.NewComicClient( + httpclient.WithHTTPClient(server.Client()), + httpclient.WithRetry(0, 0), + ) + + opts := &config.Options{ + URL: server.URL + "/title/series-1/naruto", + Country: "en", + Source: "mangadex.org", + Logger: logger.NewLogger(false, nil), + Client: client, + } + + md := NewMangadex(opts) + md.apiBase = server.URL + md.chapterBase = server.URL + "/chapter" + md.uploadsBase = server.URL + "/data" + + cleanup := func() { + server.Close() + } + + return md, cleanup +} + +func TestMangadexRetrieveIssueLinks(t *testing.T) { + md, cleanup := newTestMangadex(t) + defer cleanup() + + md.options.All = true + + links, err := md.RetrieveIssueLinks() + require.NoError(t, err) + require.Equal(t, []string{md.chapterBase + "/chapter-1"}, links) +} + +func TestMangadexInitialize(t *testing.T) { + md, cleanup := newTestMangadex(t) + defer cleanup() + + comic := &core.Comic{URLSource: md.chapterBase + "/chapter-1"} + err := md.Initialize(comic) + require.NoError(t, err) + require.Equal(t, []string{ + md.uploadsBase + "/HASH/001.png", + md.uploadsBase + "/HASH/002.png", + }, comic.Links) +} + +func TestMangadexGetInfo(t *testing.T) { + md, cleanup := newTestMangadex(t) + defer cleanup() + + title, chapter := md.GetInfo(md.chapterBase + "/chapter-1") + require.Equal(t, "Test Manga", title) + require.Equal(t, "Vol 1 Chapter 1, Start", chapter) +} diff --git a/pkg/sites/mangakakalot_test.go b/pkg/sites/mangakakalot_test.go index 8e91110b..351709ee 100644 --- a/pkg/sites/mangakakalot_test.go +++ b/pkg/sites/mangakakalot_test.go @@ -1,59 +1,95 @@ package sites import ( + "fmt" + "net/http" + "net/http/httptest" + "strings" "testing" - "github.com/Girbons/comics-downloader/pkg/core" - "github.com/Girbons/comics-downloader/internal/logger" "github.com/Girbons/comics-downloader/pkg/config" - "github.com/stretchr/testify/assert" + "github.com/Girbons/comics-downloader/pkg/core" + "github.com/stretchr/testify/require" ) -func TestMangaKakalotGetInfo(t *testing.T) { - opt := &config.Options{ - URL: "https://mangakakalot.com/read-wa8ap158524529412", - All: false, - Last: false, - Debug: false, - Logger: logger.NewLogger(false, make(chan string)), - } - mk := NewMangaKakalot(opt) - // mangakakalot.com - name, issueNumber := mk.GetInfo("https://mangakakalot.com/chapter/evergreen/chapter_2") - assert.Equal(t, "A Case", name) - assert.Equal(t, "2", issueNumber) -} +const ( + mangaKakalotChapterPath = "/chapter/manga-title/chapter-2" + mangaKakalotListPath = "/manga/manga-title" +) -func TestMangaKakalotSetup(t *testing.T) { - opt := &config.Options{ - URL: "https://mangakakalot.com/read-wa8ap158524529412", - All: false, - Last: false, - Debug: false, - Logger: logger.NewLogger(false, make(chan string)), - } - mk := NewMangaKakalot(opt) - comic := new(core.Comic) - comic.URLSource = "https://mangakakalot.com/chapter/evergreen/chapter_2" +func newMangaKakalotServer() *httptest.Server { + chapterHTML := ` + + +
+

+ Home + Chapter 2 : My Manga +

+
+
+ + +
+ + ` - err := mk.Initialize(comic) + listHTMLTemplate := ` + + +
+
+ +
+
+ +
+
+ + ` - assert.Nil(t, err) - assert.Equal(t, 40, len(comic.Links)) + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.URL.Path == mangaKakalotChapterPath: + _, _ = fmt.Fprint(w, chapterHTML) + case r.URL.Path == mangaKakalotListPath: + base := "http://" + r.Host + _, _ = fmt.Fprintf(w, listHTMLTemplate, base, base) + case strings.HasPrefix(r.URL.Path, "/chapter/manga-title/chapter-1"): + _, _ = fmt.Fprint(w, "") + default: + http.NotFound(w, r) + } + })) } -func TestMangaKakalotRetrieveIssueLinks(t *testing.T) { - opt := &config.Options{ - URL: "https://mangakakalot.com/read-wa8ap158524529412", - All: false, - Last: false, - Debug: false, - Logger: logger.NewLogger(false, make(chan string)), +func TestMangaKakalotScraper(t *testing.T) { + server := newMangaKakalotServer() + defer server.Close() + + opts := &config.Options{ + URL: server.URL + mangaKakalotListPath, + Source: "mangakakalot.com", + Logger: logger.NewLogger(false, nil), } - mk := NewMangaKakalot(opt) - links, err := mk.RetrieveIssueLinks() + scraper := NewMangaKakalot(opts) + + title, issue := scraper.GetInfo(server.URL + mangaKakalotChapterPath) + require.Equal(t, "My Manga", title) + require.Equal(t, "2", issue) + + comic := &core.Comic{URLSource: server.URL + mangaKakalotChapterPath} + require.NoError(t, scraper.Initialize(comic)) + require.Equal(t, []string{ + "https://cdn.example.com/manga-title/001.jpg", + "https://cdn.example.com/manga-title/002.jpg", + }, comic.Links) - assert.Nil(t, err) - assert.Equal(t, 46, len(links)) + links, err := scraper.RetrieveIssueLinks() + require.NoError(t, err) + require.Equal(t, []string{ + server.URL + mangaKakalotChapterPath, + server.URL + "/chapter/manga-title/chapter-1", + }, links) } diff --git a/pkg/sites/manganato_test.go b/pkg/sites/manganato_test.go index a5866720..23a85a6d 100644 --- a/pkg/sites/manganato_test.go +++ b/pkg/sites/manganato_test.go @@ -1,59 +1,93 @@ package sites import ( + "fmt" + "net/http" + "net/http/httptest" "testing" - "github.com/Girbons/comics-downloader/pkg/core" - "github.com/Girbons/comics-downloader/internal/logger" "github.com/Girbons/comics-downloader/pkg/config" - "github.com/stretchr/testify/assert" + "github.com/Girbons/comics-downloader/pkg/core" + "github.com/stretchr/testify/require" ) -func TestManganatoGetInfo(t *testing.T) { - opt := &config.Options{ - URL: "https://chapmanganato.com/manga-ng952689", - All: false, - Last: false, - Debug: false, - Logger: logger.NewLogger(false, make(chan string)), - } - mg := NewManganato(opt) - // readmanganato.com - name, issueNumber := mg.GetInfo("https://chapmanganato.com/manga-ng952689/chapter-700.5") - assert.Equal(t, "Uzumaki Naruto", name) - assert.Equal(t, "700.5", issueNumber) -} +const ( + manganatoChapterPath = "/chapter/manga-title/chapter-2" + manganatoListPath = "/read/manga-title" +) -func TestManganatoSetup(t *testing.T) { - opt := &config.Options{ - URL: "https://chapmanganato.com/manga-ng952689", - All: false, - Last: false, - Debug: false, - Logger: logger.NewLogger(false, make(chan string)), - } - mk := NewManganato(opt) - comic := new(core.Comic) - comic.URLSource = "https://chapmanganato.com/manga-ng952689/chapter-700.5" +func newManganatoServer() *httptest.Server { + chapterHTML := ` + + +
+ Home + Chapter 2 : My Manga +
+
+ + +
+ + ` - err := mk.Initialize(comic) + listHTMLTemplate := ` + + +
+
  • + +
    • +
  • + +
    • +
    + + ` - assert.Nil(t, err) - assert.Equal(t, 18, len(comic.Links)) + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case manganatoChapterPath: + _, _ = fmt.Fprint(w, chapterHTML) + case manganatoListPath: + base := "http://" + r.Host + _, _ = fmt.Fprintf(w, listHTMLTemplate, base, base) + case "/chapter/manga-title/chapter-1": + _, _ = fmt.Fprint(w, "") + default: + http.NotFound(w, r) + } + })) } -func TestManganatoRetrieveIssueLinks(t *testing.T) { - opt := &config.Options{ - URL: "https://chapmanganato.com/manga-ng952689", - All: false, - Last: false, - Debug: false, - Logger: logger.NewLogger(false, make(chan string)), +func TestManganatoScraper(t *testing.T) { + server := newManganatoServer() + defer server.Close() + + opts := &config.Options{ + URL: server.URL + manganatoListPath, + Source: "manganato.com", + Logger: logger.NewLogger(false, nil), } - mk := NewManganato(opt) - links, err := mk.RetrieveIssueLinks() - assert.Nil(t, err) - assert.Equal(t, 748, len(links)) + scraper := NewManganato(opts) + + title, issue := scraper.GetInfo(server.URL + manganatoChapterPath) + require.Equal(t, "My Manga", title) + require.Equal(t, "2", issue) + + comic := &core.Comic{URLSource: server.URL + manganatoChapterPath} + require.NoError(t, scraper.Initialize(comic)) + require.Equal(t, []string{ + "https://cdn.example.com/manga-title/001.jpg", + "https://cdn.example.com/manga-title/002.jpg", + }, comic.Links) + + links, err := scraper.RetrieveIssueLinks() + require.NoError(t, err) + require.Equal(t, []string{ + server.URL + manganatoChapterPath, + server.URL + "/chapter/manga-title/chapter-1", + }, links) } diff --git a/pkg/sites/mangareader_test.go b/pkg/sites/mangareader_test.go index 8cbb6542..9b521c9c 100644 --- a/pkg/sites/mangareader_test.go +++ b/pkg/sites/mangareader_test.go @@ -1,103 +1,93 @@ package sites -//import ( -// "testing" -// -// "github.com/Girbons/comics-downloader/internal/logger" -// "github.com/Girbons/comics-downloader/pkg/config" -// "github.com/Girbons/comics-downloader/pkg/core" -// "github.com/stretchr/testify/assert" -//) -// -//const ( -// URL = "https://www.mangareader.net/naruto/1/" -// SOURCE = "www.mangareader.net" -//) -// -//func TestMangareaderGetInfo(t *testing.T) { -// mr := new(Mangareader) -// name, issueNumber := mr.GetInfo(URL) -// -// assert.Equal(t, "naruto", name) -// assert.Equal(t, "1", issueNumber) -//} -// -//func TestRetrieveMangareaderImageLinks(t *testing.T) { -// opt := -// &config.Options{ -// URL: URL, -// Source: SOURCE, -// All: false, -// Last: false, -// Debug: false, -// Logger: logger.NewLogger(false, make(chan string)), -// } -// mr := NewMangareader(opt) -// -// comic := new(core.Comic) -// comic.URLSource = URL -// comic.Name = "naruto" -// comic.IssueNumber = "1" -// comic.Source = SOURCE -// -// links, err := mr.retrieveImageLinks(comic) -// -// assert.Equal(t, 53, len(links)) -// assert.Nil(t, err) -//} -// -//func TestSetupMangareader(t *testing.T) { -// opt := -// &config.Options{ -// URL: URL, -// Source: SOURCE, -// All: false, -// Last: false, -// Debug: false, -// Logger: logger.NewLogger(false, make(chan string)), -// } -// mr := NewMangareader(opt) -// -// comic := new(core.Comic) -// comic.Name = "naruto" -// comic.IssueNumber = "1" -// comic.URLSource = URL -// comic.Source = SOURCE -// -// err := mr.Initialize(comic) -// -// assert.Nil(t, err) -// assert.Equal(t, 53, len(comic.Links)) -//} -// -//func TestMangareaderRetrieveIssueLinks(t *testing.T) { -// opt := -// &config.Options{ -// URL: "https://www.mangareader.net/naruto", -// All: false, -// Last: false, -// Debug: false, -// Logger: logger.NewLogger(false, make(chan string)), -// } -// mr := NewMangareader(opt) -// issues, err := mr.RetrieveIssueLinks() -// -// assert.Nil(t, err) -// assert.Equal(t, 700, len(issues)) -//} -// -//func TestMangareaderRetrieveLastIssueLink(t *testing.T) { -// opt := -// &config.Options{ -// URL: "https://www.mangareader.net/naruto", -// All: false, -// Last: true, -// Debug: false, -// Logger: logger.NewLogger(false, make(chan string)), -// } -// mr := NewMangareader(opt) -// issue, err := mr.retrieveLastIssue("https://www.mangareader.net/naruto") -// -// assert.Nil(t, err) -// assert.Equal(t, "https://www.mangareader.net/naruto/700", issue) -//} +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/Girbons/comics-downloader/internal/logger" + "github.com/Girbons/comics-downloader/pkg/config" + "github.com/Girbons/comics-downloader/pkg/core" + "github.com/stretchr/testify/require" +) + +const ( + mangareaderIssuePath = "/naruto/1/" + mangareaderBasePath = "/naruto" + mangareaderLastIssue = "/naruto/700" +) + +func newMangareaderServer() *httptest.Server { + issueHTML := ` + + + + + + ` + + baseHTML := ` + + + + + +
    Chapter 1
    Chapter 2
    + + + ` + + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case mangareaderIssuePath: + _, _ = w.Write([]byte(issueHTML)) + case "/naruto/2/": + _, _ = w.Write([]byte(issueHTML)) + case mangareaderBasePath: + _, _ = w.Write([]byte(baseHTML)) + case mangareaderLastIssue: + _, _ = w.Write([]byte("")) + default: + http.NotFound(w, r) + } + })) +} + +func TestMangareaderScraper(t *testing.T) { + server := newMangareaderServer() + defer server.Close() + + opts := &config.Options{ + URL: server.URL + mangareaderIssuePath, + Logger: logger.NewLogger(false, nil), + } + + scraper := NewMangareader(opts) + + comic := &core.Comic{URLSource: server.URL + mangareaderIssuePath} + require.NoError(t, scraper.Initialize(comic)) + require.Equal(t, []string{ + "https://cdn.example.com/naruto/001.jpg", + "https://cdn.example.com/naruto/002.jpg", + }, comic.Links) + + opts.All = true + opts.URL = server.URL + mangareaderIssuePath + scraper = NewMangareader(opts) + issues, err := scraper.RetrieveIssueLinks() + require.NoError(t, err) + require.Equal(t, []string{ + "https://mangareader.tv" + mangareaderIssuePath, + "https://mangareader.tv/naruto/2/", + }, issues) + + opts.Last = true + opts.All = false + opts.URL = server.URL + mangareaderBasePath + scraper = NewMangareader(opts) + lastIssues, err := scraper.RetrieveIssueLinks() + require.NoError(t, err) + require.Equal(t, []string{"https://mangareader.tv" + mangareaderLastIssue}, lastIssues) +} diff --git a/pkg/sites/mangatown_test.go b/pkg/sites/mangatown_test.go index d4f9ef14..79f4815e 100644 --- a/pkg/sites/mangatown_test.go +++ b/pkg/sites/mangatown_test.go @@ -1,77 +1,105 @@ package sites import ( + "fmt" + "net/http" + "net/http/httptest" "testing" "github.com/Girbons/comics-downloader/internal/logger" "github.com/Girbons/comics-downloader/pkg/config" "github.com/Girbons/comics-downloader/pkg/core" - "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) -func TestMangatownGetInfo(t *testing.T) { - opt := - &config.Options{ - URL: "http://www.mangatown.com/manga/naruto/v63/c684/", - All: false, - Last: false, - Debug: false, - Logger: logger.NewLogger(false, make(chan string)), - } - mt := NewMangatown(opt) - name, issueNumber := mt.GetInfo("http://www.mangatown.com/manga/naruto/v63/c684/") +const ( + mangatownIssuePath = "/manga/naruto/v63/c684/" +) - assert.Equal(t, "naruto", name) - assert.Equal(t, "c684", issueNumber) -} +func newMangatownServer() *httptest.Server { + firstPage := ` + + +
    + +
    +
    + + ` -func TestMangatownSetup(t *testing.T) { - opt := - &config.Options{ - URL: "http://www.mangatown.com/manga/naruto/v63/c684/", - All: false, - Last: false, - Debug: false, - Logger: logger.NewLogger(false, make(chan string)), - } - mt := NewMangatown(opt) - comic := new(core.Comic) - comic.URLSource = "http://www.mangatown.com/manga/naruto/v63/c684/" + secondPage := ` + + +
    + + ` - err := mt.Initialize(comic) + listHTML := ` + + + + + ` - assert.Nil(t, err) - assert.Equal(t, 22, len(comic.Links)) + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case mangatownIssuePath: + _, _ = fmt.Fprint(w, firstPage) + case "/manga/naruto/v63/c684/1.html": + _, _ = fmt.Fprint(w, firstPage) + case "/manga/naruto/v63/c684/2.html": + _, _ = fmt.Fprint(w, secondPage) + case "/manga/naruto/v63": + _, _ = fmt.Fprint(w, listHTML) + case "/manga/naruto": + _, _ = fmt.Fprint(w, listHTML) + case "/manga/naruto/": + _, _ = fmt.Fprint(w, listHTML) + default: + http.NotFound(w, r) + } + })) } -func TestMangatownRetrieveIssueLinks(t *testing.T) { - opt := - &config.Options{ - URL: "http://www.mangatown.com/manga/naruto/v63/c684/", - All: true, - Last: false, - Debug: false, - Logger: logger.NewLogger(false, make(chan string)), - } - mt := NewMangatown(opt) - issues, err := mt.RetrieveIssueLinks() +func TestMangatownScraper(t *testing.T) { + server := newMangatownServer() + defer server.Close() - assert.Nil(t, err) - assert.Equal(t, 752, len(issues)) -} + opts := &config.Options{ + URL: server.URL + mangatownIssuePath, + Logger: logger.NewLogger(false, nil), + } -func TestMangatownRetrieveIssueLinksLastChapter(t *testing.T) { - opt := - &config.Options{ - URL: "http://www.mangatown.com/manga/naruto/", - All: false, - Last: true, - Debug: false, - Logger: logger.NewLogger(false, make(chan string)), - } - mt := NewMangatown(opt) - issues, err := mt.RetrieveIssueLinks() + scraper := NewMangatown(opts) + + comic := &core.Comic{URLSource: server.URL + mangatownIssuePath} + require.NoError(t, scraper.Initialize(comic)) + require.Equal(t, []string{ + "https://cdn.example.com/naruto/001.jpg", + "https://cdn.example.com/naruto/002.jpg", + }, comic.Links) + + opts.All = true + scraper = NewMangatown(opts) + issues, err := scraper.RetrieveIssueLinks() + require.NoError(t, err) + require.Equal(t, []string{ + "https://mangatown.com/manga/naruto/v63/c684/", + "https://mangatown.com/manga/naruto/v63/c685/", + }, issues) - assert.Nil(t, err) - assert.Equal(t, 1, len(issues)) + opts.All = false + opts.Last = true + opts.URL = server.URL + "/manga/naruto/" + scraper = NewMangatown(opts) + last, err := scraper.RetrieveIssueLinks() + require.NoError(t, err) + require.Equal(t, []string{"https://www.mangatown.com/manga/naruto/v63/c684/"}, last) } diff --git a/pkg/sites/readallcomics.go b/pkg/sites/readallcomics.go index 678ea349..c1795284 100644 --- a/pkg/sites/readallcomics.go +++ b/pkg/sites/readallcomics.go @@ -36,9 +36,12 @@ func (r *Readallcomics) retrieveImageLinks(comic *core.Comic) ([]string, error) images := document.FindAll("img") for _, img := range images { - url := img.Attrs()["src"] - if util.IsURLValid(url) { - links = append(links, url) + src, ok := img.Attrs()["src"] + if !ok { + continue + } + if util.IsURLValid(src) && !util.IsValueInSlice(src, links) { + links = append(links, src) } } @@ -61,22 +64,77 @@ func (r *Readallcomics) getIssues(url string) ([]string, error) { doc := soup.HTMLParse(response) if strings.Contains(url, "category") { - chapters := doc.Find("ul", "class", "list-story").FindAll("a") - for _, chapter := range chapters { - issueUrl := chapter.Attrs()["href"] - if util.IsURLValid(issueUrl) { - links = append(links, issueUrl) - } + chapterList := doc.Find("ul", "class", "list-story") + if chapterList.Error != nil { + return nil, fmt.Errorf("readallcomics: unable to find chapter list on %s", url) + } + for _, chapter := range chapterList.FindAll("a") { + issueURL, ok := chapter.Attrs()["href"] + if !ok { + continue + } + issueURL = strings.TrimSpace(issueURL) + if issueURL == "" { + continue + } + if !strings.HasPrefix(issueURL, "http") { + issueURL = fmt.Sprintf("%s/%s", strings.TrimRight(DefaultUrl, "/"), strings.TrimLeft(issueURL, "/")) + } + if util.IsURLValid(issueURL) && !util.IsValueInSlice(issueURL, links) { + links = append(links, issueURL) + } } } else { - chapters := doc.Find("select", "id", "selectbox").FindAll("option") - for _, chapter := range chapters { - issueUrl := chapter.Attrs()["value"] - if util.IsURLValid(issueUrl) { - links = append(links, issueUrl) + selectBox := doc.Find("select", "id", "selectbox") + if selectBox.Error == nil { + for _, chapter := range selectBox.FindAll("option") { + issueURL, ok := chapter.Attrs()["value"] + if !ok { + continue + } + issueURL = strings.TrimSpace(issueURL) + if issueURL == "" { + continue + } + if !strings.HasPrefix(issueURL, "http") { + issueURL = fmt.Sprintf("%s/%s", strings.TrimRight(DefaultUrl, "/"), strings.TrimLeft(issueURL, "/")) + } + if util.IsURLValid(issueURL) && !util.IsValueInSlice(issueURL, links) { + links = append(links, issueURL) + } } } + + if len(links) == 0 { + // fallback: scan for anchors that point to issues + for _, chapter := range doc.FindAll("a") { + issueURL, ok := chapter.Attrs()["href"] + if !ok { + continue + } + issueURL = strings.TrimSpace(issueURL) + if issueURL == "" { + continue + } + if !strings.HasPrefix(issueURL, "http") { + issueURL = fmt.Sprintf("%s/%s", strings.TrimRight(DefaultUrl, "/"), strings.TrimLeft(issueURL, "/")) + } + if !strings.Contains(issueURL, DefaultUrl) { + continue + } + if strings.Contains(issueURL, "/category/") { + continue + } + if util.IsURLValid(issueURL) && !util.IsValueInSlice(issueURL, links) { + links = append(links, issueURL) + } + } + } + + if len(links) == 0 { + return nil, fmt.Errorf("readallcomics: unable to find issue references on %s", url) + } } if r.options.Debug { @@ -103,6 +161,10 @@ func (r *Readallcomics) RetrieveIssueLinks() ([]string, error) { return nil, err } + if len(chapters) == 0 { + return nil, fmt.Errorf("readallcomics: no chapters found at %s", url) + } + if r.options.Last { return []string{chapters[len(chapters)-1]}, nil } diff --git a/pkg/sites/readallcomics_test.go b/pkg/sites/readallcomics_test.go index feed8a1d..340eb795 100644 --- a/pkg/sites/readallcomics_test.go +++ b/pkg/sites/readallcomics_test.go @@ -1,171 +1,110 @@ package sites import ( + "net/http" + "net/http/httptest" "testing" "github.com/Girbons/comics-downloader/internal/logger" "github.com/Girbons/comics-downloader/pkg/config" "github.com/Girbons/comics-downloader/pkg/core" - "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) -func TestReadallcomicsSetup(t *testing.T) { - comic := new(core.Comic) - comic.URLSource = "https://readallcomics.com/sandman-v2-075-1989/" - opt := &config.Options{ - URL: "https://readallcomics.com/sandman-v2-075-1989/", - All: false, - Last: false, - Debug: false, - Logger: logger.NewLogger(false, make(chan string)), - } - readallcomics := NewReadallcomics(opt) - err := readallcomics.Initialize(comic) - assert.Nil(t, err) - // Note: This would depend on the actual page content, adjust expected count as needed - assert.Greater(t, len(comic.Links), 0) -} - -func TestReadallcomicsGetInfoSomethingIsKillingTheChildren(t *testing.T) { - opt := &config.Options{ - URL: "https://readallcomics.com/something-is-killing-the-children-000-2024/", - All: false, - Last: false, - Debug: false, - Logger: logger.NewLogger(false, make(chan string)), - } - readallcomics := NewReadallcomics(opt) - name, issueNumber := readallcomics.GetInfo("https://readallcomics.com/something-is-killing-the-children-000-2024/") - assert.Equal(t, "something is killing the children", name) - assert.Equal(t, "000-2024", issueNumber) -} - -func TestReadallcomicsGetInfoEmbeddedIssue(t *testing.T) { - opt := &config.Options{ - URL: "https://readallcomics.com/something-is-killing-the-children-029something-is-killing-the-children-2023/", - All: false, - Last: false, - Debug: false, - Logger: logger.NewLogger(false, make(chan string)), - } - readallcomics := NewReadallcomics(opt) - name, issueNumber := readallcomics.GetInfo("https://readallcomics.com/something-is-killing-the-children-029something-is-killing-the-children-2023/") - assert.Equal(t, "something is killing the children", name) - assert.Equal(t, "029", issueNumber) -} - -func TestReadallcomicsGetInfoSandmanV2(t *testing.T) { - opt := &config.Options{ - URL: "https://readallcomics.com/sandman-v2-075-1989/", - All: false, - Last: false, - Debug: false, - Logger: logger.NewLogger(false, make(chan string)), - } - readallcomics := NewReadallcomics(opt) - name, issueNumber := readallcomics.GetInfo("https://readallcomics.com/sandman-v2-075-1989/") - assert.Equal(t, "sandman", name) - assert.Equal(t, "v2-075-1989", issueNumber) -} - -func TestReadallcomicsGetInfoSandmanDeluxeEdition(t *testing.T) { - opt := &config.Options{ - URL: "https://readallcomics.com/sandman-v2-_the_deluxe_edition-5-part-6-1989/", - All: false, - Last: false, - Debug: false, - Logger: logger.NewLogger(false, make(chan string)), - } - readallcomics := NewReadallcomics(opt) - name, issueNumber := readallcomics.GetInfo("https://readallcomics.com/sandman-v2-_the_deluxe_edition-5-part-6-1989/") - assert.Equal(t, "sandman", name) - assert.Equal(t, "v2-_the_deluxe_edition-5-part-6-1989", issueNumber) -} +const ( + readAllIssuePath = "/sandman-v2-075-1989/" + readAllIssueAlt = "/sandman-v2-_the_deluxe_edition-5-part-6-1989/" + readAllCategory = "/category/sandman/" +) -func TestReadallcomicsRetrieveIssueLinks(t *testing.T) { - opt := &config.Options{ - URL: "https://readallcomics.com/sandman-v2-075-1989/", - All: false, - Last: false, - Debug: false, - Logger: logger.NewLogger(false, make(chan string)), - } - readallcomics := NewReadallcomics(opt) - issues, err := readallcomics.RetrieveIssueLinks() - assert.Nil(t, err) - assert.Equal(t, 1, len(issues)) - assert.Equal(t, "https://readallcomics.com/sandman-v2-075-1989/", issues[0]) +func newReadAllComicsServer() *httptest.Server { + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + base := "http://" + r.Host + switch r.URL.Path { + case readAllIssuePath, readAllIssueAlt: + html := ` + + + + + + + ` + _, _ = w.Write([]byte(html)) + case readAllCategory: + html := ` + + + + + ` + _, _ = w.Write([]byte(html)) + default: + http.NotFound(w, r) + } + })) } -func TestReadallcomicsRetrieveIssueLinksFromSandmanCategory(t *testing.T) { - opt := &config.Options{ - URL: "http://readallcomics.com/category/sandman/", - All: true, - Last: false, - Debug: false, - Logger: logger.NewLogger(false, make(chan string)), - } - readallcomics := NewReadallcomics(opt) - issues, err := readallcomics.RetrieveIssueLinks() - assert.Nil(t, err) - assert.Greater(t, len(issues), 1) +func TestReadAllComicsScraper(t *testing.T) { + server := newReadAllComicsServer() + defer server.Close() - // Check if our test URLs are present - expectedURLs := []string{ - "https://readallcomics.com/sandman-v2-075-1989/", - "https://readallcomics.com/sandman-v2-_the_deluxe_edition-5-part-6-1989/", + opts := &config.Options{ + URL: server.URL + readAllIssuePath, + Logger: logger.NewLogger(false, nil), } - issueSet := make(map[string]bool) - for _, issue := range issues { - issueSet[issue] = true - } - - for _, expectedURL := range expectedURLs { - assert.True(t, issueSet[expectedURL], "Expected URL %s should be found in issues", expectedURL) - } + scraper := NewReadallcomics(opts) + + comic := &core.Comic{URLSource: server.URL + readAllIssuePath} + require.NoError(t, scraper.Initialize(comic)) + require.Equal(t, []string{ + "https://cdn.example.com/sandman/001.jpg", + "https://cdn.example.com/sandman/002.jpg", + }, comic.Links) + + // Category listing for All + opts.All = true + opts.URL = server.URL + readAllCategory + scraper = NewReadallcomics(opts) + issues, err := scraper.RetrieveIssueLinks() + require.NoError(t, err) + require.Equal(t, []string{ + server.URL + readAllIssuePath, + server.URL + readAllIssueAlt, + }, issues) + + // Last issue from category + opts.All = false + opts.Last = true + scraper = NewReadallcomics(opts) + last, err := scraper.RetrieveIssueLinks() + require.NoError(t, err) + require.Equal(t, []string{server.URL + readAllIssueAlt}, last) } -func TestReadallcomicsRetrieveIssueLinksLastFromSandman(t *testing.T) { - opt := &config.Options{ - URL: "http://readallcomics.com/category/sandman/", - All: false, - Last: true, - Debug: false, - Logger: logger.NewLogger(false, make(chan string)), - } - readallcomics := NewReadallcomics(opt) - issues, err := readallcomics.RetrieveIssueLinks() - assert.Nil(t, err) - assert.Equal(t, 1, len(issues)) - // Should return the last issue from the sandman category -} - -func TestReadallcomicsGetIssuesFromSandmanCategory(t *testing.T) { - opt := &config.Options{ - URL: "http://readallcomics.com/category/sandman/", - All: false, - Last: false, - Debug: false, - Logger: logger.NewLogger(false, make(chan string)), - } - readallcomics := NewReadallcomics(opt) - issues, err := readallcomics.getIssues("http://readallcomics.com/category/sandman/") - assert.Nil(t, err) - assert.Greater(t, len(issues), 0) - - // Check if expected URLs are present in the results - expectedURLs := []string{ - "https://readallcomics.com/sandman-v2-075-1989/", - "https://readallcomics.com/sandman-v2-_the_deluxe_edition-5-part-6-1989/", - } - - issueSet := make(map[string]bool) - for _, issue := range issues { - issueSet[issue] = true +func TestReadAllComicsGetInfoParsing(t *testing.T) { + scraper := NewReadallcomics(&config.Options{}) + + tests := []struct { + url string + expectedName string + expectedIssue string + }{ + {"https://readallcomics.com/something-is-killing-the-children-000-2024/", "something is killing the children", "000-2024"}, + {"https://readallcomics.com/sandman-v2-075-1989/", "sandman", "v2-075-1989"}, + {"https://readallcomics.com/sandman-v2-_the_deluxe_edition-5-part-6-1989/", "sandman", "v2-_the_deluxe_edition-5-part-6-1989"}, } - for _, expectedURL := range expectedURLs { - assert.True(t, issueSet[expectedURL], "Expected URL %s should be found in issues", expectedURL) + for _, tc := range tests { + name, issue := scraper.GetInfo(tc.url) + require.Equal(t, tc.expectedName, name) + require.Equal(t, tc.expectedIssue, issue) } } diff --git a/pkg/sites/readcomiconline.go b/pkg/sites/readcomiconline.go deleted file mode 100644 index d0602ae4..00000000 --- a/pkg/sites/readcomiconline.go +++ /dev/null @@ -1,181 +0,0 @@ -package sites - -import ( - "encoding/base64" - "fmt" - "regexp" - "strings" - - "github.com/Girbons/comics-downloader/pkg/config" - "github.com/Girbons/comics-downloader/pkg/core" - "github.com/Girbons/comics-downloader/pkg/util" - "github.com/anaskhan96/soup" -) - -var baseUrl = "https://readcomiconline.li" - -// ReadComicOnline represents a readcomiconline instance. -type ReadComicOnline struct { - options *config.Options -} - -// NewReadComiconline returns a readcomiconline instance. -func NewReadComiconline(options *config.Options) *ReadComicOnline { - return &ReadComicOnline{ - options: options, - } -} - -func deobfuscateUrl(imageLink string) (string, error) { - imageLink = strings.ReplaceAll(imageLink, "_x236", "d") - imageLink = strings.ReplaceAll(imageLink, "_x945", "g") - - if strings.HasPrefix(imageLink, "https://2.bp.blogspot.com") { - return imageLink, nil - } - - var quality string - - if strings.Contains(imageLink, "=s0?") { - imageLink = imageLink[:strings.Index(imageLink, "=s0?")] - quality = "=s0" - } else { - imageLink = imageLink[:strings.Index(imageLink, "=s1600?")] - quality = "=s1600" - } - - imageLink = imageLink[4:22] + imageLink[25:] - imageLink = imageLink[0:len(imageLink)-6] + imageLink[len(imageLink)-2:] - - sd, err := base64.StdEncoding.DecodeString(imageLink) - if err != nil { - return "", err - } - - imageLink = string(sd) - imageLink = imageLink[0:13] + imageLink[17:] - imageLink = imageLink[0 : len(imageLink)-2] - imageLink = imageLink + quality - - link := "https://2.bp.blogspot.com/" + imageLink - return link, nil -} - -func (c *ReadComicOnline) retrieveImageLinks(comic *core.Comic) ([]string, error) { - var links []string - - comic.URLSource = strings.Split(comic.URLSource, "?")[0] - - response, err := soup.Get(comic.URLSource + "?quality=hd&readType=1") - if err != nil { - return nil, err - } - - re := regexp.MustCompile(`push\(\'(.*?)\'\)`) - match := re.FindAllStringSubmatch(response, -1) - - for i := range match { - url := match[i][1] - - clearUrl, err := deobfuscateUrl(url) - if err != nil { - return links, err - } - - if util.IsURLValid(clearUrl) { - links = append(links, clearUrl) - } - } - - if c.options.Debug { - c.options.Logger.Debug(fmt.Sprintf("Image Links found: %s", strings.Join(links, " "))) - } - - return links, err -} - -func (c *ReadComicOnline) isSingleIssue(url string) bool { - parts := util.TrimAndSplitURL(url) - return len(parts) > 5 && strings.Contains(parts[5], "Issue-") -} - -func (c *ReadComicOnline) retrieveLastIssue(url string) (string, error) { - var lastIssue string - - response, err := soup.Get(url) - if err != nil { - return "", err - } - - name := util.TrimAndSplitURL(url)[4] - re := regexp.MustCompile("]+href=\"([^\">]+" + "/" + name + "/.+)\"") - match := re.FindAllStringSubmatch(response, -1) - lastIssue = baseUrl + strings.Split(match[0][1], "?")[0] - - return lastIssue, nil -} - -// RetrieveIssueLinks gets a slice of urls for all issues in a comic -func (c *ReadComicOnline) RetrieveIssueLinks() ([]string, error) { - url := c.options.URL - - if c.options.Last { - issue, err := c.retrieveLastIssue(url) - return []string{issue}, err - } - - if c.options.All && c.isSingleIssue(url) { - url = baseUrl + "/Comic/" + util.TrimAndSplitURL(url)[3] - } else if c.isSingleIssue(url) { - return []string{url}, nil - } - - name := util.TrimAndSplitURL(url)[4] - var ( - pages []string - links []string - ) - - response, err := soup.Get(url) - if err != nil { - return nil, err - } - - pages = append(pages, url) - re := regexp.MustCompile("]+href=\"([^\">]+" + "/" + name + "/.+)\"") - match := re.FindAllStringSubmatch(response, -1) - - for i := range match { - url := match[i][1] - if !util.IsValueInSlice(url, pages) { - url = baseUrl + strings.Split(url, "?")[0] - if util.IsURLValid(url) && !util.IsValueInSlice(url, links) { - links = append(links, url) - } - } - } - - if c.options.Debug { - c.options.Logger.Debug(fmt.Sprintf("Issues Links retrieved: %s", strings.Join(links, " "))) - } - - return links, err -} - -// GetInfo extracts the basic info from the given url. -func (c *ReadComicOnline) GetInfo(url string) (string, string) { - parts := util.TrimAndSplitURL(url) - name := parts[4] - issueNumber := strings.Split(strings.ReplaceAll(parts[5], "Issue-", ""), "?")[0] - - return name, issueNumber -} - -// Initialize will initialize the comic based -// on ReadComicOnline.to -func (c *ReadComicOnline) Initialize(comic *core.Comic) error { - links, err := c.retrieveImageLinks(comic) - comic.Links = links - - return err -} diff --git a/pkg/sites/readcomiconline_test.go b/pkg/sites/readcomiconline_test.go deleted file mode 100644 index 2e573079..00000000 --- a/pkg/sites/readcomiconline_test.go +++ /dev/null @@ -1,94 +0,0 @@ -package sites - -import ( - "testing" - - "github.com/Girbons/comics-downloader/internal/logger" - "github.com/Girbons/comics-downloader/pkg/config" - "github.com/Girbons/comics-downloader/pkg/core" - "github.com/stretchr/testify/assert" -) - -func TestReadComicOnlineSetup(t *testing.T) { - - comic := new(core.Comic) - comic.URLSource = "https://readcomiconline.li/Comic/Batman-2016/Issue-58?id=143175" - - opt := - &config.Options{ - URL: "https://readcomiconline.li/Comic/Batman-2016/Issue-58?id=143175", - All: false, - Last: false, - Debug: false, - Logger: logger.NewLogger(false, make(chan string)), - } - readComicOnline := NewReadComiconline(opt) - err := readComicOnline.Initialize(comic) - - assert.Nil(t, err) - assert.Equal(t, 24, len(comic.Links)) -} - -func TestReadComicOnlineGetInfo(t *testing.T) { - opt := - &config.Options{ - URL: "https://readcomiconline.li/Comic/Batman-2016/Issue-58?id=143175", - All: false, - Last: false, - Debug: false, - Logger: logger.NewLogger(false, make(chan string)), - } - readComicOnline := NewReadComiconline(opt) - name, issueNumber := readComicOnline.GetInfo("https://readcomiconline.li/Comic/Batman-2016/Issue-58?id=143175") - - assert.Equal(t, "Batman-2016", name) - assert.Equal(t, "58", issueNumber) -} - -func TestReadComicOnlineRetrieveIssueLinks(t *testing.T) { - opt := - &config.Options{ - URL: "https://readcomiconline.li/Comic/100-Bullets", - All: false, - Last: false, - Debug: false, - Logger: logger.NewLogger(false, make(chan string)), - } - readComicOnline := NewReadComiconline(opt) - issues, err := readComicOnline.RetrieveIssueLinks() - - assert.Nil(t, err) - assert.Equal(t, 100, len(issues)) -} - -func TestReadComicOnlineRetrieveIssueLinksLastChapter(t *testing.T) { - opt := - &config.Options{ - URL: "https://readcomiconline.li/Comic/100-Bullets", - All: false, - Last: true, - Debug: false, - Logger: logger.NewLogger(false, make(chan string)), - } - readComicOnline := NewReadComiconline(opt) - issues, err := readComicOnline.RetrieveIssueLinks() - - assert.Nil(t, err) - assert.Equal(t, 1, len(issues)) -} - -func TestReadComicOnlineRetrieveLastIssueLink(t *testing.T) { - opt := - &config.Options{ - URL: "https://readcomiconline.li/Comic/100-Bullets", - All: false, - Last: true, - Debug: false, - Logger: logger.NewLogger(false, make(chan string)), - } - readComicOnline := NewReadComiconline(opt) - issue, err := readComicOnline.retrieveLastIssue("https://readcomiconline.li/Comic/100-Bullets") - - assert.Nil(t, err) - assert.Equal(t, "https://readcomiconline.li/Comic/100-Bullets/Issue-100-2", issue) -} diff --git a/pkg/sites/readcomicsonline.go b/pkg/sites/readcomicsonline.go new file mode 100644 index 00000000..2d847f58 --- /dev/null +++ b/pkg/sites/readcomicsonline.go @@ -0,0 +1,246 @@ +package sites + +import ( + "encoding/json" + "fmt" + "regexp" + "strings" + + "github.com/Girbons/comics-downloader/pkg/config" + "github.com/Girbons/comics-downloader/pkg/core" + "github.com/Girbons/comics-downloader/pkg/util" + "github.com/anaskhan96/soup" +) + +// readComicsOnlineBaseURL is the canonical root of the site. +const readComicsOnlineBaseURL = "https://readcomicsonline.ru" + +// ReadComicsOnline represents a readcomicsonline.ru scraper instance. +type ReadComicsOnline struct { + options *config.Options + baseURL string // overridable for tests +} + +// NewReadComicsOnline returns a ReadComicsOnline instance. +func NewReadComicsOnline(options *config.Options) *ReadComicsOnline { + return &ReadComicsOnline{ + options: options, + baseURL: readComicsOnlineBaseURL, + } +} + +// pageEntry mirrors one element of the JS `var pages = [...]` array. +type pageEntry struct { + PageImage string `json:"page_image"` +} + +// retrieveImageLinks fetches the issue page and extracts all image URLs. +// +// Primary strategy – collect `data-src` attributes on +// elements (lazy-load pattern used by the site). +// +// Fallback strategy – if the primary pass yields nothing, parse the inline JS +// `var pages = [{"page_image":"01.jpg",...}]` variable and reconstruct absolute +// URLs from the known path pattern: +// +// {baseURL}/uploads/manga/{slug}/chapters/{issue}/{filename} +func (r *ReadComicsOnline) retrieveImageLinks(comic *core.Comic) ([]string, error) { + response, err := soup.Get(comic.URLSource) + if err != nil { + return nil, fmt.Errorf("readcomicsonline: fetch %s: %w", comic.URLSource, err) + } + + links := r.extractDataSrcLinks(response) + + if len(links) == 0 { + links, err = r.extractFromPagesVar(response, comic.URLSource) + if err != nil { + return nil, err + } + } + + if r.options.Debug && r.options.Logger != nil { + r.options.Logger.Debugf("readcomicsonline: found %d image links for %s", len(links), comic.URLSource) + } + + return links, nil +} + +// extractDataSrcLinks parses the HTML and collects trimmed `data-src` values +// from every element. +func (r *ReadComicsOnline) extractDataSrcLinks(html string) []string { + var links []string + doc := soup.HTMLParse(html) + + for _, img := range doc.FindAll("img", "class", "img-responsive") { + attrs := img.Attrs() + src, ok := attrs["data-src"] + if !ok { + // some pages serve the real src directly when JS is disabled + src, ok = attrs["src"] + if !ok { + continue + } + } + src = strings.TrimSpace(src) + if util.IsURLValid(src) && !util.IsValueInSlice(src, links) { + links = append(links, src) + } + } + + return links +} + +// pagesVarRe matches: var pages = [...]; +var pagesVarRe = regexp.MustCompile(`var\s+pages\s*=\s*(\[.*?\]);`) + +// extractFromPagesVar parses the inline JS `var pages` array and reconstructs +// absolute image URLs from the slug and issue number embedded in the issue URL. +func (r *ReadComicsOnline) extractFromPagesVar(html, issueURL string) ([]string, error) { + m := pagesVarRe.FindStringSubmatch(html) + if m == nil { + return nil, nil // not an error – page may just have no images yet + } + + var entries []pageEntry + if err := json.Unmarshal([]byte(m[1]), &entries); err != nil { + return nil, fmt.Errorf("readcomicsonline: parse pages var: %w", err) + } + + // derive slug and issue number from the URL + // URL shape: {base}/comic/{slug}/{issue} + parts := util.TrimAndSplitURL(issueURL) + if len(parts) < 2 { + return nil, fmt.Errorf("readcomicsonline: cannot derive slug/issue from %s", issueURL) + } + slug := parts[len(parts)-2] + issue := parts[len(parts)-1] + + base := fmt.Sprintf("%s/uploads/manga/%s/chapters/%s/", r.baseURL, slug, issue) + + var links []string + for _, e := range entries { + if e.PageImage == "" { + continue + } + url := base + e.PageImage + if !util.IsValueInSlice(url, links) { + links = append(links, url) + } + } + + return links, nil +} + +// isSingleIssue returns true when the URL points to a specific issue number, +// e.g. /comic/some-slug/1 (has a numeric final segment). +func (r *ReadComicsOnline) isSingleIssue(url string) bool { + parts := util.TrimAndSplitURL(url) + if len(parts) < 2 { + return false + } + last := parts[len(parts)-1] + return isNumeric(last) +} + +// retrieveIssueListFromComicPage fetches the comic's landing page and collects +// all issue URLs from the