diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml index ed98fc5..11bf318 100644 --- a/.github/workflows/python-publish.yml +++ b/.github/workflows/python-publish.yml @@ -1,4 +1,4 @@ -name: Deploy and Release +name: Publish on: push: @@ -29,30 +29,24 @@ jobs: with: python-version: "3.x" + - name: Install uv + uses: astral-sh/setup-uv@v6 + with: + version: "latest" + - name: Install dependencies run: | - python -m pip install --upgrade pip setuptools wheel - pip install build twine - pip install -r requirements.txt + make setup + uv pip install twine - name: Build the package - run: python -m build --no-isolation + run: uv build --link-mode copy - name: List dist directory run: ls -l dist/ - - name: Create Release - uses: ncipollo/release-action@v1 - with: - artifacts: dist/*.tar.gz,dist/*.whl - tag: ${{ github.ref_name }} - name: Release ${{ github.ref_name }} - draft: false - prerelease: false - - name: Check distribution's long description rendering on PyPI - run: twine check dist/* + run: uv run twine check dist/* - name: Publish package - if: github.repository == 'xMohnad/MangaDM' uses: pypa/gh-action-pypi-publish@release/v1 diff --git a/.python-version b/.python-version deleted file mode 100644 index e4fba21..0000000 --- a/.python-version +++ /dev/null @@ -1 +0,0 @@ -3.12 diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index e50ae22..0000000 --- a/MANIFEST.in +++ /dev/null @@ -1,3 +0,0 @@ -graft src/mangadm -global-exclude *.py[cod] -global-exclude __pycache__/ diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..a9469db --- /dev/null +++ b/Makefile @@ -0,0 +1,96 @@ +lib := mangadm +src := src/ +run := uv run +sync := uv sync +build := uv build +python := $(run) python +ptpython := $(run) ptpython +ruff := $(run) ruff +lint := $(ruff) check --select I +fmt := $(ruff) format +basedpyright := $(run) basedpyright +spell := $(run) codespell + +############################################################################## +# Local "interactive testing" of the code. +.PHONY: run +run: # Run the code with debug enabled + $(python) -m $(lib) -h + + +############################################################################## +# Setup/update packages the system requires. +.PHONY: setup +setup: # Set up the repository for development + uv venv --allow-existing + $(sync) + +.PHONY: update +update: # Update all dependencies + $(sync) --upgrade + +.PHONY: resetup +resetup: realclean # Recreate the virtual environment from scratch + make setup + +############################################################################## +# Checking/testing/linting/etc. +.PHONY: lint +lint: # Check the code for linting issues + $(lint) $(src) + +.PHONY: codestyle +codestyle: # Is the code formatted correctly? + $(fmt) --check $(src) + +.PHONY: typecheck +typecheck: # Perform static type checks with basedpyright + $(basedpyright) $(src) + +.PHONY: spellcheck +spellcheck: # Spell check the code + $(spell) *.md $(src) + +.PHONY: checkall +checkall: spellcheck codestyle lint stricttypecheck # Check all the things + +############################################################################## +# Package +.PHONY: package +package: # Package the library + $(build) + +.PHONY: spackage +spackage: # Create a source package for the library + $(build) --sdist + +############################################################################## +# Utility. +.PHONY: repl +repl: # Start a ptPython REPL in the venv. + $(run) $(ptpython) + +.PHONY: delint +delint: # Fix linting issues. + $(lint) --fix $(src) + +.PHONY: pep8ify +pep8ify: # Reformat the code to be as PEP8 as possible. + $(fmt) $(src) + +.PHONY: tidy +tidy: delint pep8ify # Tidy up the code, fixing lint and format issues. + +.PHONY: clean +clean: # Clean the package building files + rm -rf dist/ build/ $(src)*.egg-info .pytest_cache + find . -name "__pycache__" -type d -exec rm -rf {} + 2>/dev/null || true + find . -name "*.pyc" -delete + +.PHONY: realclean +realclean: clean # Clean the venv and build directories + rm -rf .venv + +.PHONY: help +help: # Display this help + @grep -Eh "^[a-z]+:.+# " $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.+# "}; {printf "%-20s %s\n", $$1, $$2}' diff --git a/README.md b/README.md index 816e702..747de9b 100644 --- a/README.md +++ b/README.md @@ -3,171 +3,24 @@ [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](./LICENSE) [![Ask DeepWiki](https://deepwiki.com/badge.svg)](https://deepwiki.com/xMohnad/MangaDM) [![PyPI Version](https://img.shields.io/pypi/v/MangaDM.svg)](https://pypi.python.org/pypi/MangaDM) +[![Python Version](https://img.shields.io/pypi/pyversions/MangaDM.svg)](https://pypi.python.org/pypi/MangaDM) -**MangaDM** is a command-line tool and Python library for downloading manga chapters based on the metadata specified in JSON files. +**MangaDM** is a powerful command-line tool for downloading manga from JSON metadata file. -## Table of Contents - -- [Installation](#installation) - - - [Install from PyPI](#install-from-pypi) - - [Install from Source](#install-from-source) - -- [CLI Usage](#cli-usage) - - - [Commands](#commands) - - [download](#download) - - [configure](#configure) - - [example](#example) - - [view](#view) - - [reset](#reset) - - [tui](#tui) - - [completion](#completion) - - [General Notes](#general-notes) - -- [Library Usage](#library-usage) - - - [Example](#example-usage) - - [Parameters](#parameters) - -- [JSON Structure](#json-structure) - - - [Structure Breakdown](#structure-breakdown) - -- [Contributing](#contributing) - - - [Bug Reports & Feature Requests](#bug-reports-and-feature-requests) - -______________________________________________________________________ +--- ## Installation -Ensure you have Python installed before proceeding. - -______________________________________________________________________ - -### Install from PyPI - ```sh pip install MangaDM ``` -______________________________________________________________________ - -### Install from Source - -To install MangaDM directly from the source code: - -1. **Clone the Repository** - - ```sh - git clone https://github.com/xMohnad/MangaDM.git - cd MangaDM - ``` - -1. **Install MangaDM** - - ```sh - pip install . - ``` - -______________________________________________________________________ - -## CLI Usage - -Once installed, you can use the `mangadm` command to download manga chapters. - -### Commands - -#### `download` - -- **Purpose**: Downloads manga chapters based on a provided JSON file. -- **Arguments**: - - `json_file`: Path to the JSON file containing manga download details. -- **Options**: - - `--dest, -p`: Destination path for downloading manga (default is the current directory). - - `--limit, -l`: Number of chapters to download (default is `-1` for all chapters). - - `--delete/--no-delete, -d`: Delete data after successful download. - - `--format, -f`: Format for downloaded manga (choices are based on `FormatType` enum). - - `--update-details/--no-update-details, -u`: Update `details.json` and re-download cover. - -#### `configure` - -- **Purpose**: Opens a configuration UI for setting up MangaDM options. - -#### `example` - -- **Purpose**: Displays an example JSON structure. - -#### `view` - -- **Purpose**: Views the current configuration settings. - -#### `reset` - -- **Purpose**: Reset saved settings. - -#### `tui` +## Command Line Interface -- **Purpose**: Open Textual TUI. - -### `completion` - -- **purpose**: the `completion` command group manages shell completion scripts for the cli tool. - -- **subcommands**: - - - `install`: installs shell autocompletion script for the specified shell. - - - **arguments**: - - `shell`: the shell type to install completion for (e.g., bash, zsh, fish). - - - `show`: displays the autocompletion script for inspection without installing. - - - **arguments**: - - `shell`: the shell type to show completion script for. - - - `shells`: lists all supported shell types for which completion is available. - -### General Notes - -- **Versioning**: The version is displayed with the `--version, -V` flag. -- **More**: Use `--help` flag in any of the commands. - -______________________________________________________________________ - -## Library Usage - -You can also use MangaDM as a Python library. Here’s how you can use it programmatically: - -### Example Usage - -```python -from mangadm import MangaDM, FormatType - -# Create an instance of MangaDM with desired parameters -mangadm = MangaDM( - json_file="path/to/yourfile.json", - dest_path="path/to/chapter", - limit=1, - delete_on_success=False, - format=FormatType.cbz, -) - -# Start the downloading process -mangadm.start() +```bash +mangadm --help ``` -### Parameters - -- `json_file` (Path): The path to the JSON file containing manga data. -- `dest_path` (str): The destination path where manga chapters will be downloaded. Defaults to the current directory. -- `limit` (int): Number of chapters to download. If -1, download all chapters. Defaults to -1. -- `delete_on_success` (bool): If `True`, delete chapter data from JSON after successful download. Defaults to False. -- `format` (FormatType): The format in which to save the downloaded manga. - -______________________________________________________________________ - ## JSON Structure Below is an example of the JSON structure required for the input file: @@ -175,17 +28,17 @@ Below is an example of the JSON structure required for the input file: ```json { "details": { - "source": "Example Source Name", - "manganame": "Example Manga Name", + "source": "Source Name", + "title": "Manga Name", "cover": "https://example.com/cover.jpg", - "description": "Example Description", - "genre": ["genre 1", "genre 2", "etc"], - "author": "Akutami Gege", - "artist": "Akutami Gege" + "description": "Description", + "genres": ["genre 1", "genre 2", "etc"], + "author": "string|null", + "artist": "string|null" }, "chapters": [ { - "title": "chapter 256 - Example Title", + "title": "chapter 256 - Title", "images": [ "https://example.com/image1.jpg", "https://example.com/image2.jpg", @@ -195,17 +48,7 @@ Below is an example of the JSON structure required for the input file: ] }, { - "title": "chapter 257 - Example Title", - "images": [ - "https://example.com/image1.jpg", - "https://example.com/image2.jpg", - "https://example.com/image3.jpg", - "https://example.com/image4.jpg", - "etc" - ] - }, - { - "title": "chapter 258 - Example Title", + "title": "chapter 257 - Title", "images": [ "https://example.com/image1.jpg", "https://example.com/image2.jpg", @@ -218,28 +61,6 @@ Below is an example of the JSON structure required for the input file: } ``` -The JSON structure provided above illustrates the format expected for input files used by the MangaDM tool. Each JSON file should follow these guidelines: - -### Structure Breakdown - -- **`details`**: Contains metadata about the manga series. - - **`source`**: The source or website where the manga is found. - - **`manganame`**: The name of the manga series. - - **`cover`**: A URL pointing to the cover image of the manga. This is optional but can enhance the metadata. - - **`description`**: A short description of the manga. - - **`genre`**: A list of genres the manga belongs to. - - **`author`**: The name of the author of the manga. - - **`artist`**: The name of the artist responsible for the manga's artwork. -- **`chapters`**: An array of objects, each representing a chapter in the manga. - - **`title`**: The title or number of the chapter. - - **`images`**: A list of URLs where each URL points to an image file, typically representing the pages of the manga chapter. - -______________________________________________________________________ - ## Contributing -### Bug Reports and Feature Requests - -If you encounter any issues while using MangaDM or have suggestions for new features, -you can open an issue on the GitHub repository. -Please provide detailed information about the issue or feature request to help in resolving it more effectively. +Contributions are welcome! If you'd like to contribute, please fork the repository and submit a pull request. For major changes, please open an issue first to discuss the proposed changes. diff --git a/pyproject.toml b/pyproject.toml index 1e76b12..d2d4051 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,20 +1,20 @@ [project] name = "MangaDM" -dynamic = ["version"] -description = "A command-line tool and Python library for downloading manga chapters based on the metadata specified in a JSON file." +version = "0.6.0" +description = "A tool for downloading manga." authors = [{ name = "xMohnad" }] readme = "README.md" -license = { file = "LICENSE" } -requires-python = ">=3.12" +license = "MIT" +requires-python = ">=3.10" dependencies = [ "aiohttp>=3.12.13", - "click>=8.2.1", - "click-completion>=0.5.2", + "dacite>=1.9.2", "ebooklib>=0.19", "inquirerpy>=0.3.4", + "pyyaml>=6.0.2", "rich>=14.0.0", - "trogon>=0.6.0", + "typer>=0.19.2", ] classifiers = [ @@ -22,36 +22,33 @@ classifiers = [ "Development Status :: 4 - Beta", "Intended Audience :: Developers", "Operating System :: OS Independent", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", "Topic :: Terminals", - "Topic :: Software Development :: Libraries", "Typing :: Typed", ] -keywords = ["terminal"] - [project.urls] -Repository = "https://github.com/xMohnad/MangaDM" -Source = "https://github.com/xMohnad/MangaDM" +Homepage = "https://github.com/xMohnad/MangaDM" Issues = "https://github.com/xMohnad/MangaDM/issues" -Discussions = "https://github.com/xMohnad/MangaDM/discussions" -[tool.setuptools] -include-package-data = true +[build-system] +requires = ["uv_build>=0.8.11,<0.10.0"] +build-backend = "uv_build" + +[project.scripts] +mangadm = "mangadm.cli:app" [tool.uv] package = true -[build-system] -requires = ["setuptools>=61.0.0", "wheel"] -build-backend = "setuptools.build_meta" - -[tool.setuptools.dynamic] -version = { attr = "mangadm.__version__" } +[tool.pyright] +reportUnusedCallResult = false -[tool.hatch.build.targets.wheel] -packages = ["src/mangadm"] +[tool.ruff] +line-length = 120 -[project.scripts] -mangadm = "mangadm.cli.main:cli" +[dependency-groups] +dev = ["codespell>=2.4.1", "ruff>=0.12.9", "ptpython>=3.0.0"] diff --git a/src/mangadm/__init__.py b/src/mangadm/__init__.py index b3867fc..24d6c5f 100644 --- a/src/mangadm/__init__.py +++ b/src/mangadm/__init__.py @@ -1,25 +1,8 @@ -from typing import TYPE_CHECKING +from importlib.metadata import version +from pathlib import Path -if TYPE_CHECKING: - from typing import Any +import typer - from .components.types import FormatType as FormatTypeType - from .core.mangadm import MangaDM as MangaDMType - -MangaDM: "type[MangaDMType]" -FormatType: "type[FormatTypeType]" - -__version__ = "0.6.0" -__all__ = ["MangaDM", "FormatType"] - - -def __getattr__(name: str) -> "Any": - if name == "MangaDM": - from .core.mangadm import MangaDM - - return MangaDM - if name == "FormatType": - from .components.types import FormatType - - return FormatType - raise AttributeError(f"module '{__name__}' has no attribute '{name}'") +__version__ = version(__name__) +__config__: Path = Path(typer.get_app_dir(__name__)) / "config.yaml" +__config__.parent.mkdir(parents=True, exist_ok=True) diff --git a/src/mangadm/__main__.py b/src/mangadm/__main__.py index ac9f281..492d501 100644 --- a/src/mangadm/__main__.py +++ b/src/mangadm/__main__.py @@ -1,4 +1,4 @@ -from mangadm.cli.main import cli +from mangadm.cli import app if __name__ == "__main__": - cli() + app() diff --git a/src/mangadm/archiver.py b/src/mangadm/archiver.py new file mode 100644 index 0000000..204c424 --- /dev/null +++ b/src/mangadm/archiver.py @@ -0,0 +1,90 @@ +# pyright: reportUnknownMemberType=false + +from __future__ import annotations + +import logging +import shutil +import zipfile +from pathlib import Path +from typing import Callable + +from ebooklib import epub # pyright: ignore[reportMissingTypeStubs] + +from mangadm.assets import build_chapter_content +from mangadm.schema.formats import FormatType + +IMAGE_EXTENSIONS: list[str] = [".jpg", ".jpeg", ".gif", ".tiff", ".tif", ".png"] + + +class MangaArchiver: + """Utility class to create manga archives in different formats (CBZ, EPUB).""" + + logger: logging.Logger = logging.getLogger(__name__) + + def get_image_paths(self, folder_path: Path) -> list[Path]: + """Return sorted list of image paths from the given folder.""" + return sorted([file for file in folder_path.absolute().rglob("*") if file.suffix.lower() in IMAGE_EXTENSIONS]) + + def _make_file(self, folder: Path, format: FormatType) -> Path: + """Return a Path for the output archive file based on format.""" + return folder.with_suffix(f".{format.value}") + + def create_cbz(self, folder_path: Path, **_: object) -> None: + """Create a CBZ (Comic Book ZIP) file from a folder and delete the original folder.""" + folder = folder_path.resolve() + cbz_file = self._make_file(folder, FormatType.cbz) + try: + with zipfile.ZipFile(cbz_file, "w", zipfile.ZIP_DEFLATED) as cbz: + for file_path in self.get_image_paths(folder): + cbz.write(file_path, file_path.relative_to(folder)) + self.logger.debug("Added image '%s' to CBZ", file_path.name) + + shutil.rmtree(folder) + self.logger.info("CBZ archive '%s' created successfully and folder removed", cbz_file.name) + except (IOError, OSError): + self.logger.exception(f"Error creating `{cbz_file.name}`") + + def create_epub(self, folder_path: Path, title: str = "", **_: object) -> None: + """Create an EPUB file from a folder of images.""" + folder = folder_path.resolve() + epub_file = self._make_file(folder, FormatType.epub) + + book = epub.EpubBook() + book.set_title(title) + + image_paths = self.get_image_paths(folder) + chapter = epub.EpubHtml( + title=title, + file_name="chapter.xhtml", + content=build_chapter_content(image_paths), + ) + book.add_item(chapter) + + # Add images as resources to the book + for image_path in image_paths: + book.add_item(epub.EpubImage(file_name=f"images/{image_path.name}", content=image_path.read_bytes())) + self.logger.debug("Added image '%s' to EPUB", image_path.name) + + book.spine = ["nav", chapter] + + try: + epub.write_epub(epub_file, book) + shutil.rmtree(folder) + self.logger.info("EPUB archive '%s' created successfully and folder removed", epub_file.name) + except (IOError, OSError) as e: + self.logger.exception("Error creating EPUB '%s': %r", epub_file.name, e) + + def create_archive(self, folder: Path, format: FormatType, title: str = "") -> None: + """Dispatch to the correct archive creation method based on format. + + Args: + folder (Path): Folder containing image files. + format (FormatType): Target archive format (e.g., cbz or epub). + title (str, optional): Optional title for formats that support it. + """ + archiver: Callable[..., None] | None = getattr(self, f"create_{format.value}", None) + if callable(archiver): + self.logger.info("Starting archive creation for '%s' as %s", folder, format.value) + archiver(folder_path=folder, title=title) + else: + self.logger.warning("No archiver found for format '%s'", format.value) diff --git a/src/mangadm/assets/__init__.py b/src/mangadm/assets/__init__.py index 30ad3f6..0ab2c69 100644 --- a/src/mangadm/assets/__init__.py +++ b/src/mangadm/assets/__init__.py @@ -1,32 +1,36 @@ from pathlib import Path -ROOT = Path(__file__).parent +ROOT: Path = Path(__file__).parent -def image_error() -> bytes: - return (ROOT / "not_available_error.webp").read_bytes() -def epub_style_css() -> bytes: - return (ROOT / "style.css").read_bytes() +def placeholder_image() -> bytes: + """Return the bytes of a default placeholder image. -def build_chapter_content(image_paths): + Used to replace missing or unavailable manga images during download. + + Returns: + bytes: Contents of the placeholder image file. + """ + return (ROOT / "placeholder_image.webp").read_bytes() + + +def build_chapter_content(image_paths: list[Path]) -> str: """Build the HTML content for the EPUB chapter with all images.""" - content = ''' + content = """ - - '''.format(style="style.css") + """ # Add image tags to the content for image_path in image_paths: - filename = Path(image_path).name + filename = image_path.name content += f'{filename}' - content += '' - return content - + content += "" + return content diff --git a/src/mangadm/assets/not_available_error.webp b/src/mangadm/assets/placeholder_image.webp similarity index 100% rename from src/mangadm/assets/not_available_error.webp rename to src/mangadm/assets/placeholder_image.webp diff --git a/src/mangadm/assets/style.css b/src/mangadm/assets/style.css deleted file mode 100644 index 82422d1..0000000 --- a/src/mangadm/assets/style.css +++ /dev/null @@ -1,12 +0,0 @@ -body, html { - margin: 0; - padding: 0; - max-width: 100%; - width: auto; -} -img { - max-width: 100%; - display: block; - margin: 0 auto; - width: auto; -} \ No newline at end of file diff --git a/src/mangadm/cli.py b/src/mangadm/cli.py new file mode 100644 index 0000000..c660c55 --- /dev/null +++ b/src/mangadm/cli.py @@ -0,0 +1,156 @@ +from __future__ import annotations + +import logging +from pathlib import Path +from typing import Annotated + +import typer + +from mangadm import __config__, __version__ +from mangadm.schema.formats import FormatType + +app: typer.Typer = typer.Typer(help="A tool for downloading manga.", rich_markup_mode="rich") + + +def version_callback(value: bool) -> None: + if value: + typer.echo(f"mangadm, version {__version__}") + raise typer.Exit() + + +@app.callback(context_settings=dict(help_option_names=["-h", "--help"])) +def common( + ctx: typer.Context, + _: Annotated[ + bool, + typer.Option( + "--version", + "-V", + help="Show the [bold cyan]version[/] and exit.", + is_eager=True, + is_flag=True, + callback=version_callback, + ), + ] = False, + verbose: Annotated[ + bool, + typer.Option( + "--verbose", + "-v", + help="Use [bold cyan]verbose[/] output", + is_flag=True, + ), + ] = False, +) -> None: + # Set logging level + logger = logging.getLogger("mangadm") + logger.setLevel(logging.DEBUG if verbose else logging.WARNING) + + import yaml + + if __config__.exists(): + with __config__.open() as f: + ctx.default_map = yaml.safe_load(f) + + +@app.command() +def download( + json_file: Annotated[ + Path, typer.Argument(..., dir_okay=False, exists=True, help="Path to manga [bold cyan]JSON[/]") + ], + dest: Annotated[ + str, + typer.Option("--dest", "-p", help="[bold cyan]dest[/]ination folder.", file_okay=False, writable=True), + ] = ".", + limit: Annotated[ + int, + typer.Option( + "--limit", "-l", help="Maximum number of chapters to download. Use -1 for no [bold cyan]limit[/].", min=-1 + ), + ] = -1, + format: Annotated[ + FormatType, + typer.Option("--format", "-f", help="Archive [bold cyan]format[/]."), + ] = FormatType.cbz, + update: Annotated[ + bool, + typer.Option( + "--update/--no-update", "-u", help="[bold cyan]Update[/] manga details and cover before downloading." + ), + ] = False, + timeout: Annotated[ + int, + typer.Option("--timeout", "-t", help="HTTP request [bold cyan]timeout[/] in seconds."), + ] = 30, + chunk_size: Annotated[ + int, + typer.Option("--chunk-size", "-c", help="[bold cyan]Chunk size[/] for each download (in bytes)."), + ] = 1024, + max_concurrent: Annotated[ + int, + typer.Option("--max-concurrent", "-m", help="Maximum number of [bold cyan]concurrent[/] downloads."), + ] = 4, + retries: Annotated[ + int, typer.Option("--retries", "-r", help="Number of [bold cyan]retries[/] for failed downloads") + ] = 3, + archive_existing: Annotated[ + bool, + typer.Option("--archive-existing/--no-archive-existing", help="[bold cyan]Archive existing[/] chapters"), + ] = False, + retry_server_errors: Annotated[ + bool, + typer.Option( + "--retry-server-errors/--no-retry-server-errors", + help="[bold cyan]Retry[/] downloads automatically when a [yellow]server error (5xx)[/] occurs.", + ), + ] = True, +) -> None: + """[bold cyan]Download[/] manga from a given JSON metadata file.""" + + from mangadm.core import Downloader + + Downloader( + json_file, + dest_path=Path(dest), + limit=limit, + format=format, + update_details=update, + timeout=timeout, + chunk_size=chunk_size, + max_concurrent=max_concurrent, + retries=retries, + archive_existing=archive_existing, + retry_server_errors=retry_server_errors, + ).start() + + +@app.command() +def show_config( + path_only: Annotated[ + bool, typer.Option("--path", "-p", help="Show only the config file [bold cyan]path[/]") + ] = False, +) -> None: + """ + [bold cyan]Show[/] the current [yellow]config[/]uration. + + Options are mapped directly to YAML keys: + + * --format -> [green]format[/green] + * --archive-existing -> [green]archive_existing[/green] + """ + if path_only: + typer.echo(__config__) + raise typer.Exit() + + import yaml + + if __config__.exists(): + with __config__.open() as f: + typer.echo(yaml.dump(yaml.safe_load(f), sort_keys=False)) + else: + typer.echo("No config file found. You can create one at:") + typer.echo(__config__) + + +if __name__ == "__main__": + app() diff --git a/src/mangadm/cli/__init__.py b/src/mangadm/cli/__init__.py deleted file mode 100644 index b9e95f3..0000000 --- a/src/mangadm/cli/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .cli_util import CliUtility, PartialMatchGroup diff --git a/src/mangadm/cli/cli_util.py b/src/mangadm/cli/cli_util.py deleted file mode 100644 index 6fbda5e..0000000 --- a/src/mangadm/cli/cli_util.py +++ /dev/null @@ -1,163 +0,0 @@ -import json -from functools import cached_property -from pathlib import Path -from typing import Any, Dict, Optional - -import click - - -class PartialMatchGroup(click.Group): - def get_command(self, ctx: click.Context, cmd_name: str) -> Optional[click.Command]: - """Finds and returns a command by name, supporting partial matches.""" - full_name = self._resolve_command_name(cmd_name) - return super().get_command(ctx, full_name) - - def _resolve_command_name(self, partial_name: str) -> str: - """Resolves a partial command name to the full command name.""" - matches = [name for name in self.commands if name.startswith(partial_name)] - - if not matches: - return partial_name # No match found, return as-is - - if len(matches) == 1: - return matches[0] # Single match found - - # Multiple matches found - raise click.BadParameter( - f"Ambiguous command '{partial_name}'. Possible matches: {', '.join(sorted(matches))}" - ) - - -class CliUtility: - SHELL_CHOICES = ["bash", "fish", "zsh", "powershell"] - - @cached_property - def console(self): - from rich.console import Console - - return Console() - - @cached_property - def config_path(self): - return self._resolve_config_path() - - @cached_property - def settings(self): - return self._load() - - @cached_property - def formats(self): - from mangadm import FormatType - - return [ft.value for ft in FormatType] - - @property - def shells(self): - return sorted(self.SHELL_CHOICES) - - def validate_shell(self, shell: Optional[str]) -> str: - """Validate and return the shell name, or auto-detect if None.""" - from click_completion import get_auto_shell - - selected_shell = shell or get_auto_shell() - if selected_shell not in self.shells: - raise click.BadParameter( - f"Unsupported shell: {selected_shell}. " - f"Available options: {', '.join(self.shells)}" - ) - return selected_shell - - def _resolve_config_path(self) -> Path: - """ - Get the path to the configuration file. - - - On Linux: ~/.config/manga_dm/config.json - - On Windows: %APPDATA%/manga_dm/config.json - - Creates the directory if it does not exist. - - Returns: - str: Absolute path to the configuration file. - """ - import os - - base_dir = Path(os.getenv("APPDATA") or Path.home() / ".config") - config_dir = base_dir / "manga_dm" - config_dir.mkdir(parents=True, exist_ok=True) - return config_dir / "config.json" - - def _load(self) -> Dict[str, Any]: - if not self.config_path.exists(): - return {} - try: - return json.loads(self.config_path.read_text(encoding="utf-8")) - except (OSError, json.JSONDecodeError) as e: - self.console.print(f"[Load error] {e}", style="red") - return {} - - def save(self, settings: Dict[str, Any]) -> None: - try: - self.config_path.write_text( - json.dumps(settings, ensure_ascii=False, indent=4), encoding="utf-8" - ) - except OSError as e: - self.console.print(f"[Save error] {e}", style="red") - - def show_example(self) -> None: - image_urls = [ - "https://example.com/image1.jpg", - "https://example.com/image2.jpg", - "https://example.com/image3.jpg", - "https://example.com/image4.jpg", - "etc", - ] - example_json = { - "details": { - "source": "Example Source Name", - "manganame": "Example Manga Name", - "cover": "https://example.com/cover.jpg", - "description": "Example Description", - "genre": ["genre 1", "genre 2", "etc"], - "author": "Akutami Gege", - "artist": "Akutami Gege", - }, - "chapters": [ - {"title": f"chapter {i} - Example Title", "images": image_urls} - for i in range(256, 259) - ], - } - self.console.print_json(json.dumps(example_json)) - - def reset(self) -> None: - from InquirerPy.resolver import prompt - - confirmation = prompt( - [ - { - "type": "confirm", - "name": "confirm_reset", - "message": "Are you sure you want to reset all settings?", - "default": False, - } - ] - ) - if not confirmation.get("confirm_reset", False): - self.console.print("[yellow]Reset cancelled.") - return - - try: - self.config_path.unlink() - self.console.print("[green]Settings have been reset.") - except FileNotFoundError: - self.console.print("[yellow]No settings to reset.") - - def display_settings(self, settings: Dict[str, Any]) -> None: - from rich.table import Table - - table = Table(title="Current MangaDM Settings") - table.add_column("Key", style="cyan") - table.add_column("Value", style="magenta") - for key, value in settings.items(): - if key != "save_defaults": - table.add_row(key, str(value)) - self.console.print(table) diff --git a/src/mangadm/cli/main.py b/src/mangadm/cli/main.py deleted file mode 100644 index a1684aa..0000000 --- a/src/mangadm/cli/main.py +++ /dev/null @@ -1,212 +0,0 @@ -from typing import Any, Dict, Optional, cast - -import click - -from mangadm import __version__ as version -from mangadm.cli import CliUtility, PartialMatchGroup - -cli_util = CliUtility() - - -@click.group( - help="A CLI tool for downloading manga chapters based on a JSON metadata file.", - context_settings={"help_option_names": ["-h", "--help"]}, - cls=PartialMatchGroup, -) -@click.version_option(version, "--version", "-V") -def cli(): - pass - - -@cli.command() -@click.argument("json_file", type=click.Path(exists=True, readable=True)) -@click.option( - "--dest", - "-p", - default=cli_util.settings.get("dest", "."), - help="Download destination.", -) -@click.option( - "--limit", - "-l", - default=cli_util.settings.get("limit", -1), - help="Limit chapters.", -) -@click.option( - "--delete/--no-delete", - "-d", - default=cli_util.settings.get("delete", False), - help="Delete after success.", -) -@click.option( - "--format", - "-f", - type=click.Choice(cli_util.formats, case_sensitive=False), - default=cli_util.settings.get("format", "cbz"), - help="Download format.", -) -@click.option( - "--update-details/--no-update-details", - "-u", - default=cli_util.settings.get("update_details", False), - help="Update details file and cover.", -) -def download(json_file, dest, limit, delete, format, update_details): - """Download manga chapters based on a JSON file.""" - from pathlib import Path - - from mangadm import FormatType, MangaDM - - MangaDM( - json_file=Path(json_file), - dest_path=dest, - limit=limit, - delete_on_success=delete, - format=FormatType(format), - update_details=update_details, - ).start() - - -@cli.command() -def configure(): - """Open configuration UI.""" - - from InquirerPy.resolver import prompt - from prompt_toolkit.completion import PathCompleter - - current = cli_util.settings - style = { - "completion-menu.completion": "bg:#444444 #ffffff", - "completion-menu.completion.current": "bg:#00afff #ffffff", - "scrollbar.background": "bg:#333333", - "scrollbar.button": "bg:#888888", - } - questions = [ - { - "type": "input", - "name": "dest", - "message": "Download destination:", - "completer": PathCompleter(only_directories=True, expanduser=True), - "validate": lambda val: len(val.strip()) > 0, - "default": str(current.get("dest", ".")), - }, - { - "type": "input", - "name": "limit", - "message": "Number of chapters (-1 = all):", - "default": str(current.get("limit", -1)), - "validate": lambda r: r == "-1" or (r.isdigit() and int(r) > 0), - "invalid_message": "Enter -1 or a positive integer.", - "filter": lambda r: int(r) if r == "-1" or r.isdigit() else -1, - }, - { - "type": "list", - "name": "format", - "message": "Choose format:", - "choices": cli_util.formats, - "default": current.get("format", "cbz"), - }, - { - "type": "confirm", - "name": "delete", - "message": "Delete JSON data after download?", - "default": current.get("delete", False), - }, - { - "type": "confirm", - "name": "update_details", - "message": "Update details and re-download cover?", - "default": current.get("update_details", False), - }, - { - "type": "confirm", - "name": "save_defaults", - "message": "Save these settings as default?", - "default": True, - }, - ] - answers = prompt(questions, style=style, style_override=False) - if answers.get("save_defaults"): - assert isinstance(answers, dict), "Expected answers to be a dict" - settings_dict = cast(Dict[str, Any], answers) - cli_util.save(settings_dict) - cli_util.display_settings(settings_dict) - click.secho("Settings saved successfully.", fg="green") - else: - click.secho("Settings not saved.", fg="yellow") - - -@cli.command() -def example(): - """Display an example JSON structure.""" - cli_util.show_example() - - -@cli.command() -def reset(): - """Reset saved settings.""" - cli_util.reset() - - -@cli.command() -def view(): - """View current settings.""" - cli_util.display_settings(cli_util.settings) - - -@cli.command(name="tui", help="Open interactive TUI.") -def run_tui(): - from trogon import tui - - tui()(cli)() - - -@cli.group(cls=PartialMatchGroup) -def completion() -> None: - """Manage shell completion scripts for this CLI tool.""" - pass - - -@completion.command(name="install") -@click.argument( - "shell", - required=False, - metavar="", - type=click.Choice(cli_util.shells), -) -def install_script(shell: Optional[str]) -> None: - """Install shell autocompletion script.""" - from click_completion import install - - selected_shell = cli_util.validate_shell(shell) - shell, path = install(selected_shell) - click.secho(f"{shell} completion installed in {path}", fg="green") - - -@completion.command() -@click.argument( - "shell", - required=False, - metavar="", - type=click.Choice(cli_util.shells), -) -def show(shell: Optional[str]) -> None: - """Show the autocompletion script for inspection.""" - from click_completion import get_code - - selected_shell = cli_util.validate_shell(shell) - click.echo(get_code(selected_shell)) - - -@completion.command() -def shells(): - """List all supported shell types for completion.""" - click.echo("Supported shell types:") - from click_completion import shells - - for shell, doc in shells.items(): - click.echo(f" - {shell}: {doc}") - - -if __name__ == "__main__": - cli() diff --git a/src/mangadm/components/__init__.py b/src/mangadm/components/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/src/mangadm/components/archiver.py b/src/mangadm/components/archiver.py deleted file mode 100644 index 4dc62e3..0000000 --- a/src/mangadm/components/archiver.py +++ /dev/null @@ -1,113 +0,0 @@ -import os -import shutil -import zipfile -from pathlib import Path -from typing import List - -from ebooklib import epub -from rich.console import Console - -from mangadm.assets import build_chapter_content, epub_style_css -from mangadm.components.base_component import BaseComponent -from mangadm.components.types import FormatType - - -class MangaArchiver(BaseComponent): - def __init__(self, console: Console) -> None: - self.SUPPORTED_IMAGE_EXTENSIONS = (".png", ".jpg", ".jpeg", ".webp") - self.console = console - - def _get_image_paths(self, folder_path: Path): - """Return sorted list of image paths from the given folder.""" - return [ - os.path.join(root, file) - for root, _, files in os.walk(folder_path) - for file in sorted(files) - if file.lower().endswith(self.SUPPORTED_IMAGE_EXTENSIONS) - ] - - def get_image_paths(self, folder_path: Path) -> List[Path]: - """Return sorted list of image paths from the given folder.""" - folder = Path(folder_path).resolve() - return sorted( - [ - file - for file in folder.rglob("*") - if file.suffix.lower() in self.SUPPORTED_IMAGE_EXTENSIONS - ] - ) - - def create_cbz(self, folder_path: Path) -> None: - """Create a CBZ (Comic Book ZIP) file from a folder and delete the original folder.""" - folder = Path(folder_path).resolve() - folder_name = folder.name - cbz_file = folder.parent / f"{folder_name}.cbz" - - try: - image_paths = self.get_image_paths(folder) - - with zipfile.ZipFile(cbz_file, "w", zipfile.ZIP_DEFLATED) as cbz: - for file_path in image_paths: - cbz.write(file_path, file_path.relative_to(folder)) - - shutil.rmtree(str(folder)) - except (IOError, OSError) as e: - self.console.log(f"Error creating `{cbz_file.name}`: {e}") - - def create_epub(self, folder_path: Path) -> None: - """Create an EPUB file from a folder of images.""" - folder = Path(folder_path).resolve() - folder_name = folder.name - epub_file = folder.parent / f"{folder_name}.epub" - - book = epub.EpubBook() - book.set_title(folder_name) - book.set_language("en") - - # Add default Navigation files - book.add_item(epub.EpubNcx()) - book.add_item(epub.EpubNav()) - - # Add stylesheet - style = epub.EpubItem( - uid="style", - file_name="style.css", - media_type="text/css", - content=epub_style_css(), - ) - book.add_item(style) - - image_paths = self.get_image_paths(folder) - - # Create the chapter content with all images - chapter_content = build_chapter_content(image_paths) - chapter = epub.EpubHtml( - title=folder_name, - file_name="chap_all_images.xhtml", - content=chapter_content, - ) - book.add_item(chapter) - - # Add images as resources to the book - for image_path in image_paths: - filename = os.path.basename(image_path) - with open(image_path, "rb") as img_file: - img_item = epub.EpubImage() - img_item.file_name = f"images/{filename}" - img_item.content = img_file.read() - book.add_item(img_item) - - # Set the table of contents and spine - book.toc = [chapter] - book.spine = ["nav", chapter] - - # Write the EPUB file - epub.write_epub(epub_file, book) - - shutil.rmtree(folder) - - def create_archiver(self, folder: Path, format: FormatType): - if format == FormatType.cbz: - self.create_cbz(folder) - elif format == FormatType.epub: - self.create_epub(folder) diff --git a/src/mangadm/components/base_component.py b/src/mangadm/components/base_component.py deleted file mode 100644 index 8188d53..0000000 --- a/src/mangadm/components/base_component.py +++ /dev/null @@ -1,18 +0,0 @@ -from typing import Optional - -from rich.console import Console - - -class BaseComponent: - """Base class for all components with shared functionality.""" - - def __init__(self, console: Optional[Console] = None): - self._console = console or Console() - - @property - def console(self) -> Console: - return self._console - - @console.setter - def console(self, value: Console): - self._console = value diff --git a/src/mangadm/components/types.py b/src/mangadm/components/types.py deleted file mode 100644 index 225fbc9..0000000 --- a/src/mangadm/components/types.py +++ /dev/null @@ -1,73 +0,0 @@ -from dataclasses import dataclass -from enum import Enum, auto -from pathlib import Path -from typing import List, Optional - - -class FormatType(str, Enum): - cbz = "cbz" - epub = "epub" - - -class DownloadStatus(Enum): - SUCCESS = auto() - FAILED = auto() - SKIPPED = auto() - REPLACED = auto() - - -@dataclass -class DownloadResult: - status: DownloadStatus - path: Optional[Path] = None - error: Optional[BaseException] = None - url: Optional[str] = None - - -class Status: - def __init__(self) -> None: - self._results: List[DownloadResult] = [] - - @property - def results(self) -> List[DownloadResult]: - """Get the download results.""" - return self._results - - def clear_results(self): - self._results.clear() - - def _get_status_count(self, status: DownloadStatus) -> int: - """Helper method to count results by status.""" - return sum(1 for r in self.results if r.status == status) - - @property - def success_count(self) -> int: - """Get count of success downloads.""" - return self._get_status_count(DownloadStatus.SUCCESS) - - @property - def failed_count(self) -> int: - """Get count of failed downloads.""" - return self._get_status_count(DownloadStatus.FAILED) - - @property - def skipped_count(self) -> int: - """Get count of skipped downloads.""" - return self._get_status_count(DownloadStatus.SKIPPED) - - @property - def replaced_count(self) -> int: - """Get count of replaced downloads.""" - return self._get_status_count(DownloadStatus.REPLACED) - - @property - def all_success(self) -> bool: - """Check if all downloads were successful.""" - if not self.results: # No downloads completed yet - return False - return all( - result.status == DownloadStatus.SUCCESS - or result.status == DownloadStatus.SKIPPED - or result.status == DownloadStatus.REPLACED - for result in self.results - ) diff --git a/src/mangadm/core.py b/src/mangadm/core.py new file mode 100644 index 0000000..33c04f8 --- /dev/null +++ b/src/mangadm/core.py @@ -0,0 +1,432 @@ +from __future__ import annotations + +import asyncio +import logging +import random +from collections.abc import Awaitable, Coroutine +from contextlib import AsyncExitStack, suppress +from functools import cached_property, wraps +from pathlib import Path +from typing import Callable, Final, TypeVar + +import aiohttp +from aiohttp import ClientSession, ClientTimeout +from rich.filesize import decimal +from rich.logging import RichHandler +from rich.progress import ( + BarColumn, + DownloadColumn, + MofNCompleteColumn, + Progress, + SpinnerColumn, + TaskID, + TaskProgressColumn, + TextColumn, + TimeElapsedColumn, + TransferSpeedColumn, +) + +from mangadm.archiver import MangaArchiver +from mangadm.schema.formats import FormatType +from mangadm.schema.manga import Chapter, Manga + +TRANSLATION_TABLE: Final[dict[int, str]] = str.maketrans( + { + "/": "_", + "\\": "_", + '"': "_", + "'": "_", + "?": "", + "*": " ", + "%": " ", + "$": "_", + "#": "_", + "@": "_", + "~": "_", + "}": "-", + "{": "-", + "|": "_", + ":": "_", + "+": "_", + "=": "_", + "[": "-", + "]": "-", + "&": "-", + } +) + +TEMP_EXT: Final[str] = "_tmp" + +RETRY_EXCEPTIONS: Final[tuple[type[Exception], ...]] = ( + aiohttp.ServerTimeoutError, + aiohttp.ClientConnectionError, + asyncio.TimeoutError, + aiohttp.ClientResponseError, +) + +T = TypeVar("T") + + +def temp_path(path: Path) -> Path: + """Return a temporary path derived from the given file or folder path.""" + return path.with_name(path.name + TEMP_EXT) + + +def get_size(local_filename: Path) -> int: + """Get the size of a file if it exists, otherwise return 0.""" + return local_filename.stat().st_size if local_filename.exists() else 0 + + +def retry_delay( + attempt: int, + base: float = 2.5, + factor: float = 2.5, + max_delay: float = 30.0, + jitter: float = 0.5, +) -> float: + """Compute exponential backoff delay with optional jitter.""" + delay = min(base * factor ** (attempt - 1), max_delay) + return max(0, delay * (1 + random.uniform(-jitter, jitter))) + + +def run_async( + func: Callable[..., Coroutine[None, None, T]], +) -> Callable[..., T | asyncio.Task[T]]: + """ + Decorator to allow calling an async function directly. + + - If called inside a running event loop, returns asyncio.Task[T]. + - If called outside an event loop, returns T (result of asyncio.run). + """ + + @wraps(func) + def wrapper(*args: object, **kwargs: object) -> T | asyncio.Task[T]: + try: + loop = asyncio.get_running_loop() + except RuntimeError: + loop = None + + if loop and loop.is_running(): + return asyncio.create_task(func(*args, **kwargs)) + else: + return asyncio.run(func(*args, **kwargs)) + + return wrapper + + +class Downloader: + """Manage the full manga download and archiving workflow. + + The class automates downloading manga chapters from URLs listed in a JSON file + or Manga object. + + + Attributes: + manga (Manga): Parsed manga metadata. + dest_path (Path): Output directory for downloads. + limit (int): Maximum chapters to process (-1 means all). + format (FormatType): Output archive format + update_details (bool): Whether to re-download metadata and cover image. + chunk_size (int): Download chunk size in bytes. + archive_existing (bool): Whether to archive existing complete chapters. + retry_server_errors (bool): Whether to retry on 5xx HTTP responses. + retries (int): Maximum number of retry attempts per request. + manga_folder (Path): Directory containing all manga chapters and files. + details_path (Path): Path to stored manga metadata JSON file. + cover_path (Path): Path to the downloaded cover image. + _progress (Progress): Progress bar for per-file downloads. + _spinner (Progress): Spinner display for chapter-level status. + _semaphore (asyncio.Semaphore): Concurrency limiter for download tasks. + _timeout (ClientTimeout): Global network timeout configuration. + _archiver (MangaArchiver): Utility for creating archive. + logger (logging.Logger): Logger configured with RichHandler. + """ + + def __init__( + self, + json: Path | Manga, + *, + dest_path: Path | None = None, + limit: int = -1, + format: FormatType = FormatType.cbz, + update_details: bool = False, + timeout: int = 30, + chunk_size: int = 1024, + max_concurrent: int = 4, + retries: int = 3, + archive_existing: bool = True, + retry_server_errors: bool = True, + ) -> None: + """Initialize MangaDM instance with configuration. + + Args: + json (Path | Manga): Manga JSON path or Manga object. + dest_path (Path | None, optional): Destination directory. Defaults to current path. + limit (int, optional): Maximum chapters to download. -1 means no limit. + format (FormatType, optional): Archive format (cbz or epub). Defaults to cbz. + update_details (bool, optional): If True, re-download manga metadata. Defaults to False. + timeout (int, optional): Network timeout in seconds. Defaults to 30. + chunk_size (int, optional): Download chunk size in bytes. Defaults to 1024. + max_concurrent (int, optional): Maximum concurrent downloads. Defaults to 4. + retries (int, optional): Maximum retry attempts per file. Defaults to 3. + archive_existing (bool, optional): If True, archive completed chapters automatically. Defaults to True. + retry_server_errors (bool, optional): Retry on server errors (5xx). Defaults to True. + """ + self.manga: Manga = json if isinstance(json, Manga) else Manga.from_json_file(json) + self.dest_path: Path = dest_path or Path(".") + self.limit: int = limit + self.format: FormatType = format + self.update_details: bool = update_details + self.chunk_size: int = chunk_size + self.archive_existing: bool = archive_existing + + # retry options + self.retry_server_errors: bool = retry_server_errors + self.retries: int = retries + + details = self.manga.details + manga_name = details.title.translate(TRANSLATION_TABLE) + + # paths + self.manga_folder: Path = self.dest_path / f"{manga_name} ({details.source})" + self.details_path: Path = self.manga_folder / "details.json" + self.cover_path: Path = self.manga_folder / "cover.png" + + self._progress: Progress = Progress( + TextColumn("[bold blue]{task.description}", justify="right"), + BarColumn(), + TransferSpeedColumn(), + DownloadColumn(), + TaskProgressColumn(), + transient=True, + ) + self._spinner: Progress = Progress( + SpinnerColumn(), + TextColumn("[bold green]{task.description}[/]"), + TextColumn("• [bold blue]Chapter {task.fields[chapter]}/{task.fields[chapters]}[/] •"), + MofNCompleteColumn(), + BarColumn(), + TaskProgressColumn(), + TimeElapsedColumn(), + transient=True, + ) + + self._semaphore: asyncio.Semaphore = asyncio.Semaphore(max_concurrent) + self._timeout: ClientTimeout = ClientTimeout(timeout) + self._archiver: MangaArchiver = MangaArchiver() + + self.logger: logging.Logger = logging.getLogger("mangadm") + self.logger.addHandler(RichHandler(show_time=False, show_path=False)) + + @cached_property + def _spinner_task(self) -> TaskID: + """Initialize spinner task for chapter-level progress.""" + return self._spinner.add_task( + "", + total=0, + chapter=0, + chapters=len(self.manga.chapters), + ) + + async def _handle_http_error(self, e: aiohttp.ClientResponseError, temp: Path, image_path: Path) -> bool: + """Handle specific HTTP errors and decide whether to retry. + + Args: + e (aiohttp.ClientResponseError): The raised HTTP exception. + temp (Path): Temporary download file path. + image_path (Path): Target image path. + + Returns: + bool: True if retry should occur, False otherwise. + """ + if e.status in (401, 403, 404): + from mangadm.assets import placeholder_image + + image_path.write_bytes(placeholder_image()) + self.logger.error("Replaced %s with placeholder (HTTP %d).", image_path.name, e.status) + return False + + if e.status == 416: + size = e.headers.get("Content-Range", "") if e.headers else "" + size = int(size.split("/")[-1]) if "/" in size else 0 + self.logger.warning( + "416 Range error for %s (server=%s, local=%s). Restarting.", + temp, + decimal(size) if size else "unknown", + decimal(get_size(temp)), + ) + with suppress(Exception): + temp.unlink() + return True + + if e.status == 429: + retry_after = e.headers.get("Retry-After") if e.headers else None + msg = f"Retry after {retry_after}" if retry_after else "Please retry later" + self.logger.warning("Too many requests. %s", msg) + raise e + + if e.status < 500 and not self.retry_server_errors: + self.logger.warning( + "HTTP %d received. Retry disabled by configuration (retry_server_errors=False).", e.status + ) + return False + + return True + + async def download(self, url: str, session: ClientSession, path: Path) -> None: + """Save manga metadata and cover image before downloading chapters. + + Writes details JSON and downloads the cover image if missing or outdated. + + Args: + session (ClientSession): Active aiohttp client session. + """ + temp = temp_path(path) + + for attempt in range(1, self.retries + 1): + task_id: None | TaskID = None + local_size = get_size(temp) + headers = {"Range": f"bytes={local_size}-"} if local_size else {} + try: + async with session.get(url, timeout=self._timeout, headers=headers) as resp: + # Reset if server does not support ranges + if resp.status != 206 and "Accept-Ranges" not in resp.headers: + local_size, headers = 0, {} + + total = int(resp.headers.get("content-length", 0)) + local_size or None + task_id = self._progress.add_task(path.name, total=total, completed=local_size) + + resp.raise_for_status() + with temp.open("ab" if local_size else "wb") as f: + async for chunk in resp.content.iter_chunked(self.chunk_size): + f.write(chunk) + self._progress.update(task_id, advance=len(chunk)) + + temp.rename(path) + return self._progress.remove_task(task_id) + + except Exception as e: + if task_id: + self._progress.remove_task(task_id) + + if attempt == self.retries: + self.logger.exception("Final attempt #%d failed: %s", attempt, path.name) + raise + + if isinstance(e, aiohttp.ClientResponseError): + if not await self._handle_http_error(e, temp, path): + raise + + elif not isinstance(e, RETRY_EXCEPTIONS): + self.logger.exception("Non-retryable exception on attempt #%d for %s", attempt, path) + raise + + delay = retry_delay(attempt) + self.logger.warning("Attempt %d on %s failed (%r). Retrying in %.2f", attempt, path.name, e, delay) + await asyncio.sleep(delay) + + async def _prepare_details(self, session: ClientSession) -> None: + """Save manga metadata and cover image before downloading chapters. + + Writes details JSON and downloads the cover image if missing or outdated. + + Args: + session (ClientSession): Active aiohttp client session. + """ + if not self.details_path.exists() or self.update_details: + self.manga.details.to_json(self.details_path) + + cover = self.manga.details.cover + cover_path = self.cover_path.with_suffix(Path(cover).suffix) + if not cover_path.exists() or self.update_details: + await self.download(cover, session, cover_path) + + async def _task(self, url: str, image_path: Path, session: ClientSession) -> None: + """Wrapper around download with semaphore limiting concurrency. + + Args: + url (str): Image URL. + image_path (Path): Output file path. + session (ClientSession): Active aiohttp client session. + """ + async with self._semaphore: + await self.download(url, session, image_path) + self._spinner.update(self._spinner_task, advance=1) + + def _skip_chapter(self, path: Path, chapter: Chapter) -> bool: + """Determine if a chapter should be skipped. + + Args: + path (Path): Chapter folder path. + chapter (Chapter): Chapter metadata. + + Returns: + bool: True if chapter should be skipped. + """ + if not chapter.images: + self.logger.warning(f"Chapter '{chapter.title}' has no images.") + return True + + archive_file = path.with_suffix(f".{self.format.value}") + if archive_file.is_file(): + self.logger.info("Skipping %s: %s already exists.", path.name, archive_file.name) + return True + + if path.exists() and len(chapter.images) == len(list(path.iterdir())): + if self.archive_existing: + self.logger.info("Chapter '%s' exists, will attempt archiving.", path.name) + self._archiver.create_archive(path, self.format, chapter.title) + return True + + return False + + @run_async + async def start(self) -> None: + """Start the manga download process.""" + self.manga_folder.mkdir(parents=True, exist_ok=True) + async with AsyncExitStack() as stack: + stack.enter_context(self._progress) + stack.enter_context(self._spinner) + session = await stack.enter_async_context(ClientSession()) + await self._prepare_details(session) + for idx, chapter in enumerate(self.manga.chapters, 1): + title = chapter.title.translate(TRANSLATION_TABLE) + chapter_path = self.manga_folder / title + temp = temp_path(chapter_path) + temp.mkdir(exist_ok=True) + + if self.limit > 0 and idx > self.limit: + self.logger.warning("Limit %d reached at '%s'", self.limit, chapter.title) + break + + if self._skip_chapter(chapter_path, chapter): + continue + + self._spinner.update( + self._spinner_task, + total=len(chapter.images), + completed=0, + description=chapter.title, + chapter=idx, + ) + + tasks: set[Awaitable[None]] = set() + for i, url in enumerate(chapter.images, 1): + image_path = temp / f"{i:02d}{Path(url).suffix}" + if image_path.exists(): + self.logger.debug("Image '%s' in %s already exists. Skipping.", image_path.name, chapter.title) + self._spinner.update(self._spinner_task, advance=1) + continue + + tasks.add(self._task(url, image_path, session)) + + if skipped := len(chapter.images) - len(tasks): + self.logger.debug("Total skipped %d existing images in chapter '%s'", skipped, chapter.title) + + errs = await asyncio.gather(*tasks, return_exceptions=True) + if any([isinstance(err, BaseException) for err in errs]): + self.logger.warning(f"Chapter '{chapter.title}' has errors. Skipping.") + continue + + temp.rename(chapter_path) + self._archiver.create_archive(chapter_path, self.format, chapter.title) diff --git a/src/mangadm/core/__init__.py b/src/mangadm/core/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/src/mangadm/core/mangadm.py b/src/mangadm/core/mangadm.py deleted file mode 100644 index 5a33b87..0000000 --- a/src/mangadm/core/mangadm.py +++ /dev/null @@ -1,265 +0,0 @@ -import json -from functools import cached_property -from pathlib import Path -from typing import Dict, List, Optional - -from rich.console import Console, Group -from rich.live import Live - -from mangadm.components.archiver import MangaArchiver -from mangadm.components.base_component import BaseComponent -from mangadm.components.types import DownloadResult, DownloadStatus, FormatType, Status -from mangadm.core.slide_loader import SlideLoader - - -class MangaDM(BaseComponent, Status): - """Manages the downloading of manga chapters from a JSON file.""" - - # fmt: off - # Prepare translation table for sanitizing folder and file names - translation_table = str.maketrans({ - "/": "_", "\\": "_", "\"": "_", "'": "_", "?": "", "*": " ", "%": " ", - "$": "_", "#": "_", "@": "_", "~": "_", "}": "-", "{": "-", "|": "_", - ":": "_", "+": "_", "=": "_", "[": "-", "]": "-", "&": " - " - }) - # fmt: on - - def __init__( - self, - json_file: Path, - dest_path: Path = Path("."), - limit: int = -1, - delete_on_success: bool = False, - update_details: bool = False, - format: FormatType = FormatType.cbz, - console: Optional[Console] = None, - ) -> None: - super().__init__(console) - self.json_file = json_file - self.dest_path = dest_path - self.limit = limit - self.delete_on_success = delete_on_success - self.update_details = update_details - self.format = format - - self._results: List[DownloadResult] = [] - - self.loader = SlideLoader(console=self.console, save_dir=self.base_folder) - self.archiver = MangaArchiver(self.console) - - @property - def json_file(self): - return self._json_file - - @json_file.setter - def json_file(self, value): - if not str(value).lower().endswith(".json"): - raise ValueError( - f"Unsupported file format: {value}. Please use JSON format." - ) - self._json_file = value - self.data = self._load_data() - - @property - def data(self) -> dict: - """Get manga data with validation.""" - if not self._data: - raise ValueError("No manga data available") - return self._data - - @data.setter - def data(self, data): - match data: - case {"details": dict(), "chapters": list()}: - self._data = data - case _: - raise ValueError( - "Invalid JSON structure. Expected format: " - "{ 'details': dict, 'chapters': list }" - ) - - @property - def dest_path(self): - return self._dest_path - - @dest_path.setter - def dest_path(self, value): - if not isinstance(value, Path): - value = Path(value) - - self._dest_path = value - details = self.data.get("details", {}) - manga_name = details.get("manganame", "UnknownManga").translate( - self.translation_table - ) - source = details.get("source", "unknown") - self.base_folder = self.dest_path / f"{manga_name} ({source})" - self.base_folder.mkdir(parents=True, exist_ok=True) - - @property - def chapters(self) -> list: - """Get chapters list with validation.""" - return self.data.get("chapters", []) - - @cached_property - def details(self) -> Dict: - """Get details.""" - return self.data.get("details", {}) - - def _should_stop_processing(self) -> bool: - """Check if processing should stop based on limit.""" - return self.limit > 0 and (self.failed_count + self.success_count) >= self.limit - - def _is_downloaded_chapter( - self, - folder: Path, - temp_folder: Path, - images: List[str], - ) -> bool: - """Check if a chapter is already downloaded using only Path operations and multiple archive extensions.""" - # Check if any archive with supported extensions exists - for ext in [f.value for f in FormatType]: - archive_file = folder.with_name(f"{folder.name}.{ext}") - if archive_file.is_file(): - return True - - # Check if the temporary folder exists - if temp_folder.is_dir(): - return False - - if not folder.is_dir(): - return False - - files = list(folder.iterdir()) - if any(self.loader.temp_ext in file.name for file in files): - return False - - return len(images) == len(files) - - def _load_data(self) -> dict: - """Load JSON data from file.""" - try: - with open(self.json_file, "r", encoding="utf-8") as f: - return json.load(f) - except (json.JSONDecodeError, IOError) as e: - self.console.log(f"Error loading JSON data: {e}") - return {"details": {}, "chapters": []} - - def _save_data(self, file_path: Path, data: Dict) -> bool: - """Save current data back to JSON file.""" - try: - with open(file_path, "w", encoding="utf-8") as f: - json.dump(data, f, ensure_ascii=False, indent=4) - return True - except (IOError, OSError) as e: - self.console.log(f"Error saving data: {e}") - return False - - def create_temp_folder(self, chapter_name: str) -> Path: - """Create temporary folder for chapter downloads.""" - temp_path = self.base_folder / f"{chapter_name}_tmp" - temp_path.mkdir(exist_ok=True) - return temp_path - - def _rename(self, temp_path: Path, final_path: Path) -> bool: - """Rename temporary folder to final chapter name.""" - try: - if final_path.exists(): - self.console.log(f"Path already exists: {final_path}") - return False - temp_path.rename(final_path) - return True - except Exception as e: - self.console.log(f"Error renaming folder: {e}") - return False - - def _remove_chapter(self, chapter_data: dict) -> bool: - """Remove chapter from data and save.""" - try: - self.data["chapters"].remove(chapter_data) - return self._save_data(self.json_file, self.data) - except ValueError as e: - self.console.log(f"Error removing chapter: {e}") - return False - - def _setup_manga_metadata(self) -> None: - """Setup manga details and cover image.""" - details_path = self.base_folder / "details.json" - details = self.details - if not details_path.exists() or self.update_details: - sanitized_details = { - "title": details.get("manganame", "UnknownManga"), - "author": details.get("author"), - "artist": details.get("artist"), - "description": details.get("description"), - "genre": details.get("genre", []), - } - self._save_data(details_path, sanitized_details) - self._download_cover_image() - - def _download_cover_image(self) -> None: - """Download manga cover image if needed.""" - cover_url = self.details.get("cover") - if not cover_url: - return - - cover_path = self.base_folder / f"cover{Path(cover_url).suffix}" - if not cover_path.exists() or self.update_details: - self.loader.one(cover_url, Path(cover_path.name)) - - def _process_chapters(self) -> None: - """Process all chapters in the manga data.""" - - chapters = self.chapters[:] - total_chapters = len(chapters) - for idx, chapter in enumerate(chapters, 1): - if self._should_stop_processing(): - break - self._process_chapter(chapter, idx, total_chapters) - - def _process_chapter( - self, chapter: dict, current_idx: int, total_chapters: int - ) -> None: - """Process a single manga chapter.""" - title = chapter.get("title", "UnknownChapter").translate(self.translation_table) - images = chapter.get("images", []) - - if not images: - self.console.log(f"Chapter [green]{title}[/] has no images.") - return - - final_path = self.base_folder / title - temp_path = self.create_temp_folder(title) - - if self._is_downloaded_chapter(final_path, temp_path, images): - self._results.append(DownloadResult(DownloadStatus.SKIPPED, final_path)) - return - - self.loader.totel = (total_chapters, self.skipped_count, current_idx) - self.loader.save_dir = temp_path - self.loader.urls = images - self.loader.all() - - if self.loader.all_success: - self.loader.clear_results() - self._results.append( - DownloadResult(DownloadStatus.SUCCESS, self.loader.save_dir) - ) - if self._rename(temp_path, final_path): - self.archiver.create_archiver(final_path, self.format) - if self.delete_on_success: - self._remove_chapter(chapter) - else: - self._results.append( - DownloadResult(DownloadStatus.FAILED, self.loader.save_dir) - ) - self.loader.spinner.remove_task(self.loader.spinner_task) - - def start(self) -> None: - """Start download process""" - with Live( - Group(self.loader.spinner, self.loader.progress), - console=self.console, - ): - self._setup_manga_metadata() - self._process_chapters() diff --git a/src/mangadm/core/slide_loader.py b/src/mangadm/core/slide_loader.py deleted file mode 100644 index 9648d0f..0000000 --- a/src/mangadm/core/slide_loader.py +++ /dev/null @@ -1,270 +0,0 @@ -import asyncio -from contextlib import asynccontextmanager -from functools import cached_property -from pathlib import Path -from typing import Dict, List, Optional, Union - -from aiohttp import ClientResponseError, ClientTimeout -from aiohttp.client import ClientSession -from rich.console import Console -from rich.progress import ( - BarColumn, - DownloadColumn, - Progress, - SpinnerColumn, - TextColumn, - TransferSpeedColumn, -) - -from mangadm.components.base_component import BaseComponent -from mangadm.components.types import DownloadResult, DownloadStatus, Status - - -class SlideLoader(BaseComponent, Status): - """Asynchronous image downloader with progress tracking.""" - - def __init__( - self, - urls: Optional[List[str]] = None, - save_dir: Union[Path, str] = Path("chapter"), - max_concurrent: int = 4, - chunk_size: int = 1024, - timeout: int = 30, - console: Optional[Console] = None, - ): - """ - Initialize the SlideLoader. - - Args: - urls: List of image URLs to download - save_dir: Directory to save downloaded images - max_concurrent: Maximum concurrent downloads - chunk_size: Size of chunks for downloading files - timeout: Timeout for HTTP requests in seconds - """ - super().__init__(console) - - self._urls = urls or [] - self.max_concurrent = max_concurrent - self.chunk_size = chunk_size - self.timeout = timeout - self.save_dir = save_dir - self._current_tasks: Dict[asyncio.Task, str] = {} # Task -> URL mapping - self._results: List[DownloadResult] = [] - - self.progress = self._progress - self.spinner = self._spinner_progress - self.temp_ext = "_tmp" - - @cached_property - def spinner_task(self): - return self.spinner.add_task( - "", - total=len(self.urls), - chapter=0, - total_chapters=0, - ) - - def get_size(self, local_filename: Path) -> int: - """Get the size of a file if it exists, otherwise return 0.""" - return local_filename.stat().st_size if local_filename.exists() else 0 - - def save_image_error(self, save_path): - from mangadm.assets import image_error - - with open(save_path, "wb") as f: - f.write(image_error()) - - @property - def save_dir(self) -> Path: - """Get the current save directory.""" - return self._save_dir - - @save_dir.setter - def save_dir(self, save_dir: Union[Path, str]): - """Set a new save directory.""" - if not isinstance(save_dir, Path): - save_dir = Path(save_dir) - save_dir.mkdir(exist_ok=True, parents=True) - self._save_dir = save_dir - - @property - def urls(self) -> List[str]: - """Get the current list of URLs.""" - return self._urls - - @urls.setter - def urls(self, urls: List[str]): - """Set a new list of URLs and cancel any ongoing downloads.""" - for task in self._current_tasks: - task.cancel() - self._urls = urls - - @property - def _spinner_progress(self) -> Progress: - return Progress( - SpinnerColumn(), - TextColumn( - "[bold blue]Chapter {task.fields[chapter]}/{task.fields[total_chapters]}" - ), - TextColumn("• Page {task.completed}/{task.total}"), - BarColumn(), - TextColumn("[progress.percentage]{task.percentage:>3.0f}%"), - expand=True, - ) - - @property - def _progress(self) -> Progress: - """Create a new progress bar instance.""" - return Progress( - TextColumn("[bold blue]{task.description}", justify="right"), - BarColumn(), - TransferSpeedColumn(), - DownloadColumn(), - transient=True, - expand=True, - ) - - @property - def totel(self): - return self._totel - - @totel.setter - def totel(self, value: tuple[int, int, int]): - self._totel = value - - def _make_name( - self, - url: str, - filename: Optional[Union[Path, str]] = None, - index: Optional[int] = None, - ) -> Path: - """Generate a filename from URL and options.""" - suffix = Path(url).suffix - if index is not None: - return Path(f"{index:02d}{suffix}") - if filename and not isinstance(filename, Path): - filename = Path(filename) - if filename: - return filename.with_suffix(suffix) - - raise ValueError("Either 'filename' or 'index' must be provided.") - - @asynccontextmanager - async def get_session(self): - async with ClientSession() as session: - yield session - - async def download( - self, - url: str, - session: ClientSession, - filename: Path, - final_filepath: Path, - ) -> DownloadResult: - """Download a single file.""" - temp_filepath = self.save_dir / f"{filename.name}{self.temp_ext}" - completed = self.get_size(temp_filepath) - headers = {"Range": f"bytes={completed}-"} if completed else {} - timeout = ClientTimeout(total=self.timeout) - id = None - try: - async with session.get(url, timeout=timeout, headers=headers) as response: - response.raise_for_status() - if response.status != 206 and "Accept-Ranges" not in response.headers: - completed = 0 - headers.pop("Range", None) - - content_length = response.headers.get("content-length") - total_size = int(content_length) + completed if content_length else None - task_id = self.progress.add_task( - str(filename), total=total_size, completed=completed - ) - id = task_id - with open(temp_filepath, "ab" if completed > 0 else "wb") as f: - async for chunk in response.content.iter_chunked(self.chunk_size): - f.write(chunk) - self.progress.update(task_id, advance=len(chunk)) - - # Rename temp file to final filename after successful download - temp_filepath.rename(final_filepath) - self.progress.remove_task(task_id) - return DownloadResult(DownloadStatus.SUCCESS, final_filepath, url=url) - - except ClientResponseError as e: - result = DownloadResult(DownloadStatus.REPLACED, temp_filepath, e, url=url) - if e.status in [401, 403, 404]: - self.save_image_error(final_filepath) - - if id is not None: - self.progress.remove_task(id) - self.console.log(result) - return result - - except Exception as e: - result = DownloadResult(DownloadStatus.FAILED, temp_filepath, e, url=url) - if id is not None: - self.progress.remove_task(id) - self.console.log(result) - return result - - async def download_all(self) -> None: - """Download all images concurrently with a limit on simultaneous downloads.""" - semaphore = asyncio.Semaphore(self.max_concurrent) - chapters, skipped_count, count = self.totel - self.spinner.update( - self.spinner_task, - total=len(self.urls), - chapter=skipped_count + count, - total_chapters=chapters, - ) - - async with self.get_session() as session: - - async def _download_task(index: int, url: str): - filename = self._make_name(url, index=index) - final_path = self.save_dir / filename - - async with semaphore: - result = await self.download(url, session, filename, final_path) - self.spinner.update(self.spinner_task, advance=1) - self._results.append(result) - - tasks = [] - for i, url in enumerate(self.urls, 1): - filename = self._make_name(url, index=i) - final_path = self.save_dir / filename - if final_path.exists(): - self._results.append( - DownloadResult( - DownloadStatus.SKIPPED, - path=final_path, - url=url, - ) - ) - continue - tasks.append(asyncio.create_task(_download_task(i, url))) - self.spinner.update( - self.spinner_task, completed=len(self.urls) - len(tasks) - ) - - await asyncio.gather(*tasks, return_exceptions=True) - - def all(self) -> List[DownloadResult]: - """Synchronously download all images.""" - asyncio.run(self.download_all()) - return self._results.copy() - - async def _one(self, url: str, filename: Path) -> DownloadResult: - """Download a single file.""" - final_filepath = self.save_dir / self._make_name(url, filename) - async with self.get_session() as session: - return await self.download(url, session, filename, final_filepath) - - def one(self, url: str, filename: Path) -> bool: - """Download a single file synchronously.""" - result = asyncio.run(self._one(url, filename)) - return ( - isinstance(result, DownloadResult) - and result.status == DownloadStatus.SUCCESS - ) diff --git a/src/mangadm/schema/formats.py b/src/mangadm/schema/formats.py new file mode 100644 index 0000000..1425b12 --- /dev/null +++ b/src/mangadm/schema/formats.py @@ -0,0 +1,26 @@ +from __future__ import annotations + +from enum import Enum + + +class FormatType(str, Enum): + """ + Enum representing supported output formats for manga archives. + + Attributes: + cbz (str): Comic Book ZIP format. + epub (str): EPUB e-book format. + """ + + cbz = "cbz" + epub = "epub" + + @classmethod + def formats(cls) -> list[str]: + """ + Return a list of all supported format values. + + Returns: + list[str]: A list of supported format strings + """ + return [ft.value for ft in cls] diff --git a/src/mangadm/schema/manga.py b/src/mangadm/schema/manga.py new file mode 100644 index 0000000..1a45b78 --- /dev/null +++ b/src/mangadm/schema/manga.py @@ -0,0 +1,49 @@ +from __future__ import annotations + +import json +from dataclasses import asdict, dataclass +from pathlib import Path + +from dacite import from_dict + + +@dataclass +class MangaDetails: + source: str + title: str + cover: str + description: str + genres: list[str] + author: str | None = None + artist: str | None = None + + def to_json(self, path: Path) -> None: + """Save manga details to a JSON file.""" + path.write_text( + json.dumps( + asdict(self), + indent=4, + ensure_ascii=False, + ), + encoding="utf-8", + ) + + +@dataclass +class Chapter: + title: str + images: list[str] + + +@dataclass +class Manga: + details: MangaDetails + chapters: list[Chapter] + + @classmethod + def from_json_file(cls, file_path: Path) -> Manga: + """Load a Manga instance from a JSON file.""" + return from_dict( + cls, + json.loads(file_path.read_text(encoding="utf-8")), # pyright: ignore[reportAny] + )