diff --git a/.github/workflows/brand.yml b/.github/workflows/brand.yml index 2fbd97f..ae62a17 100644 --- a/.github/workflows/brand.yml +++ b/.github/workflows/brand.yml @@ -10,6 +10,7 @@ on: - "scripts/brand_generate.py" - "tests/test_brand_generate.py" - "Taskfile.yml" + - "website/assets/**" - ".github/workflows/brand.yml" permissions: diff --git a/.github/workflows/deploy-docs.yml b/.github/workflows/deploy-docs.yml index a9fc084..a79b7ea 100644 --- a/.github/workflows/deploy-docs.yml +++ b/.github/workflows/deploy-docs.yml @@ -5,7 +5,7 @@ on: branches: - main paths: - - "site-docs/**" + - "website/**" - "zensical.toml" - "requirements-docs.txt" - ".github/workflows/deploy-docs.yml" diff --git a/.github/workflows/docs-quality.yml b/.github/workflows/docs-quality.yml index de2532e..46e9a4c 100644 --- a/.github/workflows/docs-quality.yml +++ b/.github/workflows/docs-quality.yml @@ -8,7 +8,7 @@ on: branches: - main paths: - - "site-docs/**" + - "website/**" - "zensical.toml" - "requirements-docs.txt" - "scripts/check_docs.py" diff --git a/README.md b/README.md index adbfba5..7604b84 100644 --- a/README.md +++ b/README.md @@ -1,91 +1,78 @@ # ffmpegfyMe -**ffmpegfyMe** is a Dockerized Python CLI tool for downloading streaming media (e.g. `.m3u8`) using `ffmpeg`. -It offers interactive or batch downloading, progress tracking, retry logic, and state persistence. +![ffmpegfyMe](brand/generated/readme-banner.svg) -[Documentation](https://fnayou.github.io/ffmpegfyMe/) +**Download HTTP/HLS streams as MP4 or MKV — no transcoding.** ---- +ffmpegfyMe is a Python CLI that wraps FFmpeg to download HTTP/HTTPS media streams, commonly HLS playlists, into local MP4 or MKV files. -## Features +## Implemented capabilities -- Stream download using `ffmpeg` -- Manual or automatic batch mode -- Progress tracking per batch -- Retry logic and skip for already-downloaded files -- Persistent state across runs -- Docker + Taskfile support -- Input via versioned JSON list file +- Interactive or batch Python CLI. +- HTTP/HTTPS inputs; playlist segments may use HTTP, HTTPS, TCP, TLS, or crypto via the FFmpeg protocol whitelist. +- MP4 or MKV outputs via FFmpeg stream copy (`-c copy`); no transcoding. +- Optional HTTP header or cookie auth uses exact `${ENV_NAME}` environment references. +- Saved per-item progress; completed items are skipped on rerun. +- Initial attempt plus up to two retries. +- 60-second default inactivity timeout, reset by FFmpeg output. No total-download timeout is implemented. ---- +Every capability listed here is implemented. Planned work is tracked in GitHub Issues and may not be available. -## Project Structure +## Prerequisites -```bash -ffmpegfyMe/ -├── ffmpegfyMe.py # Main CLI script -├── docker/ -│ └── Dockerfile # ffmpeg + Python base image -├── compose.yml # Compose service: ffmpegfy-me -├── Taskfile.yml # Task runner integration -├── .gitignore -├── requirements.txt -├── in/ -│ ├── stream-list.json # Versioned input list -│ └── .gitkeep -├── out/ -│ └── .gitkeep -└── progress/ - └── .gitkeep -``` +**Docker quick start** (for the shown `task` commands): ---- +- Docker Engine + Docker Compose +- Task -## Usage +**Direct local use (without Docker):** -### 1. Prepare your input file +- Python 3.11+ +- FFmpeg installed and on `PATH` -Example: `in/stream-list.json` +## Quick start -```json -[ - { - "url": "https://cdn.example.com/video1.m3u8", - "output": "video1.mp4", - "id": "video-1" - } -] -``` +Before running, add one or more JSON manifests to `in/`. The minimal example below shows the required fields. -The optional `id` is a public stable identifier and must not contain secrets. - -### 2. Build the container +For Docker, run: ```bash task docker:build +task ffmpegfy:run ``` -### 3. Run interactively +For direct local use from the repository root, run: ```bash -task ffmpegfy:run +python3 ffmpegfyMe.py +``` + +## Minimal manifest + +```json +[ + { + "url": "https://cdn.example.com/video1.m3u8", + "output": "video1.mp4" + } +] ``` -### 4. Configure authentication (optional) +## Authentication Protected sources may require HTTP header or cookie auth. -Supported source and HLS segment protocols are `HTTP`/`HTTPS`. FFmpeg runs with -`-protocol_whitelist http,https,tcp,tls,crypto`, so nested protocols are restricted. +Auth values use exact environment references such as `${API_TOKEN}` or `${SESSION_COOKIE}`. + +References must be exact `${ENV_NAME}` values; prefixed values such as `Bearer ${API_TOKEN}` are invalid. -Manifest example with auth env references (`in/stream-list.json`): +The optional `headers` and `cookies` fields may be omitted for unauthenticated sources. ```json [ { "url": "https://cdn.example.com/video1.m3u8", "output": "video1.mp4", - "id": "video-1", "headers": { "Authorization": "${API_TOKEN}" }, @@ -94,97 +81,29 @@ Manifest example with auth env references (`in/stream-list.json`): ] ``` -- Env references must be exactly `${ENV_NAME}` (e.g. `${API_TOKEN}`, `${SESSION_COOKIE}`). -- Values like `Bearer ${API_TOKEN}` are invalid; refs must be exact. -- Credentials do not belong in the manifest; only `${ENV_NAME}` placeholders. +Credentials live in a git-ignored `.env` and are injected via `env_file` for Docker; direct local runs need the variables exported in the process environment. -Header and cookie auth can serve playlists and segments. +Displayed FFmpeg output and error lines redact HTTP(S) URLs and configured cookie values. Header values may appear if FFmpeg echoes them. Progress state stores item status, output, and an identity key, not raw manifest URLs or credentials. -Create a git-ignored `.env` with placeholder example values: +Signed URLs remain plaintext in the manifest. Protect it, and do not share or commit URLs containing credentials. -```bash -API_TOKEN=your-api-token-here -SESSION_COOKIE=your-session-cookie-here -``` - -Compose injects these values at runtime via `env_file`. No real secrets. - -Credentials are redacted (shown as `[REDACTED]`) in CLI output, errors, FFmpeg diagnostics, progress, and summaries. +Local users or processes may inspect credential-bearing FFmpeg arguments while FFmpeg runs. -Limitation: local users/processes may inspect credential-bearing FFmpeg arguments while FFmpeg runs. +## Documentation -### FFmpeg inactivity timeout - -Default inactivity timeout is 60 seconds without FFmpeg output; any output resets the inactivity timer. - -Set positive seconds with `FFMPEG_INACTIVITY_TIMEOUT` in the git-ignored `.env`: - -```bash -FFMPEG_INACTIVITY_TIMEOUT=120 -``` - -Invalid, non-finite, zero, and negative values produce a clear error before batch processing. - -A timeout counts as a failed attempt and follows the retry policy: initial attempt plus up to two retries. - -There is no total-download time limit by default; a download can run longer while FFmpeg continues producing output. - ---- - -## Docker - -- Base image: `python:3.11-slim-bookworm` -- FFmpeg pinned to Debian Bookworm package `7:5.1.9-0+deb12u1` (ffmpeg + ffprobe report `5.1.9`) -- Source: - -### Runtime data & secrets - -- Image contains code only — no Git metadata, manifests, credentials, media, output, or progress state. -- Manifests (`in/`), credentials (`.env`), outputs (`out/`), progress state (`progress/`) are supplied at runtime via bind mounts / env file. - -```bash -task docker:build -task docker:smoke -task ffmpegfy:run -``` - -- Secrets live in `.env` (git-ignored) and are injected via `env_file` at runtime only. - ---- - -## Tests - -Run the Python test suite with branch coverage. See [CONTRIBUTING.md](CONTRIBUTING.md#tests-and-ci) for the local environment and commands. - -GitHub Actions runs it on pull requests to `main` and pushes to `main`. - ---- - -## Dependencies - -- Python 3.11+ -- `ffmpeg` installed in container -- Docker + Compose plugin -- Taskfile (https://taskfile.dev) - ---- +- [Documentation](https://fnayou.github.io/ffmpegfyMe/) +- [Getting Started](website/getting-started.md) +- [Manifest](website/manifest.md) +- [Authentication](website/authentication.md) +- [Reliability](website/reliability.md) +- [Troubleshooting](website/troubleshooting.md) ## Community -- [Code of Conduct](CODE_OF_CONDUCT.md) -- [Contributing](CONTRIBUTING.md) -- [Security](SECURITY.md) -- [Support](SUPPORT.md) - -## License +- [Contributing](CONTRIBUTING.md) +- [Support](SUPPORT.md) +- [Security](SECURITY.md) +- [Code of Conduct](CODE_OF_CONDUCT.md) +- [License](LICENSE) This project is licensed under the MIT License. See [LICENSE](LICENSE) for details. - ---- - -## Author - -```txt -Name: Aymen FNAYOU -GitHub: https://github.com/fnayou -``` diff --git a/scripts/brand_generate.py b/scripts/brand_generate.py index 802ea44..dfeb119 100644 --- a/scripts/brand_generate.py +++ b/scripts/brand_generate.py @@ -28,6 +28,7 @@ SOCIAL_PREVIEW_WIDTH = 1280 SOCIAL_PREVIEW_HEIGHT = 640 SOCIAL_PREVIEW_MAX_BYTES = 1_000_000 +WEBSITE_ASSET_NAMES = ("logo.svg", "favicon.svg") PNG_SIGNATURE = b"\x89PNG\r\n\x1a\n" PALETTE_KEYS = ("base", "mantle", "crust", "text", "subtext1", "surface0", "surface1", "sky") @@ -216,6 +217,27 @@ def write_generated_assets(root: Path, assets: dict[str, bytes]) -> None: (generated_dir / filename).write_bytes(content) +def check_website_assets(root: Path, assets: dict[str, bytes]) -> list[str]: + errors: list[str] = [] + website_assets_dir = root / "website" / "assets" + for filename in sorted(WEBSITE_ASSET_NAMES): + expected_bytes = assets[filename] + path = website_assets_dir / filename + if not path.is_file(): + errors.append(f"missing website asset: website/assets/{filename}") + continue + if path.read_bytes() != expected_bytes: + errors.append(f"stale website asset: website/assets/{filename}") + return errors + + +def write_website_assets(root: Path, assets: dict[str, bytes]) -> None: + website_assets_dir = root / "website" / "assets" + website_assets_dir.mkdir(parents=True, exist_ok=True) + for filename in sorted(WEBSITE_ASSET_NAMES): + (website_assets_dir / filename).write_bytes(assets[filename]) + + def generate_brand_assets(root: Path) -> dict[str, bytes]: palette = _read_json(root / "brand" / "data" / "brand.json") facts = _read_json(root / "brand" / "data" / "facts.json") @@ -235,6 +257,7 @@ def check_brand_assets(root: Path) -> list[str]: "social-preview.svg": render_social_preview_svg(palette, facts).encode("utf-8"), } errors = check_generated_assets(root, svg_assets) + errors.extend(check_website_assets(root, svg_assets)) generated_dir = root / "brand" / "generated" png_path = generated_dir / "social-preview.png" metadata_path = generated_dir / SOCIAL_PREVIEW_METADATA @@ -281,6 +304,7 @@ def main(argv: list[str] | None = None) -> int: if args.generate: assets = generate_brand_assets(ROOT_DIR) write_generated_assets(ROOT_DIR, assets) + write_website_assets(ROOT_DIR, assets) for filename in sorted(assets): content = assets[filename] detail = f"{len(content)} bytes" if filename.endswith(".png") else "JSON" if filename.endswith(".json") else "SVG" diff --git a/tests/test_brand_generate.py b/tests/test_brand_generate.py index ce3049a..b99de53 100644 --- a/tests/test_brand_generate.py +++ b/tests/test_brand_generate.py @@ -10,6 +10,7 @@ BrandError, check_brand_assets, check_generated_assets, + check_website_assets, render_favicon_svg, render_generated_assets, render_logo_svg, @@ -19,6 +20,7 @@ validate_palette, validate_social_preview_png, write_generated_assets, + write_website_assets, ) @@ -130,6 +132,44 @@ def test_check_detects_missing_and_stale_generated_assets(self) -> None: ], ) + def test_check_detects_missing_and_stale_website_logo_and_favicon(self) -> None: + with tempfile.TemporaryDirectory() as tmp_dir: + root = Path(tmp_dir) + data_dir = root / "brand" / "data" + data_dir.mkdir(parents=True) + (data_dir / "brand.json").write_text(json.dumps(VALID_PALETTE), encoding="utf-8") + (data_dir / "facts.json").write_text(json.dumps(VALID_FACTS), encoding="utf-8") + assets = self.render_assets_without_rsvg() + write_generated_assets(root, assets) + website_assets_dir = root / "website" / "assets" + website_assets_dir.mkdir(parents=True) + write_website_assets(root, assets) + + self.assertEqual(check_brand_assets(root), []) + + logo_path = website_assets_dir / "logo.svg" + logo_path.write_bytes(b"stale") + self.assertEqual(check_brand_assets(root), ["stale website asset: website/assets/logo.svg"]) + + favicon_path = website_assets_dir / "favicon.svg" + favicon_path.unlink() + self.assertEqual( + check_brand_assets(root), + [ + "missing website asset: website/assets/favicon.svg", + "stale website asset: website/assets/logo.svg", + ], + ) + + def test_write_website_assets_copies_canonical_brand_bytes(self) -> None: + with tempfile.TemporaryDirectory() as tmp_dir: + root = Path(tmp_dir) + assets = self.render_assets_without_rsvg() + write_website_assets(root, assets) + website_assets_dir = root / "website" / "assets" + self.assertEqual((website_assets_dir / "logo.svg").read_bytes(), assets["logo.svg"]) + self.assertEqual((website_assets_dir / "favicon.svg").read_bytes(), assets["favicon.svg"]) + def test_brand_check_uses_metadata_to_detect_stale_png_without_rendering(self) -> None: with tempfile.TemporaryDirectory() as tmp_dir: root = Path(tmp_dir) @@ -139,6 +179,7 @@ def test_brand_check_uses_metadata_to_detect_stale_png_without_rendering(self) - (data_dir / "facts.json").write_text(json.dumps(VALID_FACTS), encoding="utf-8") assets = self.render_assets_without_rsvg() write_generated_assets(root, assets) + write_website_assets(root, assets) with patch("scripts.brand_generate.render_social_preview_png", side_effect=AssertionError("renderer called")): self.assertEqual(check_brand_assets(root), []) diff --git a/website/assets/favicon.svg b/website/assets/favicon.svg new file mode 100644 index 0000000..108aa7c --- /dev/null +++ b/website/assets/favicon.svg @@ -0,0 +1,29 @@ + + ffmpegfyMe + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/website/assets/logo.svg b/website/assets/logo.svg new file mode 100644 index 0000000..3b32a44 --- /dev/null +++ b/website/assets/logo.svg @@ -0,0 +1,39 @@ + + ffmpegfyMe + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/site-docs/authentication.md b/website/authentication.md similarity index 89% rename from site-docs/authentication.md rename to website/authentication.md index bc6214b..9a98525 100644 --- a/site-docs/authentication.md +++ b/website/authentication.md @@ -35,8 +35,7 @@ A direct local Python run needs the variables exported in the process environmen ## Privacy -- The CLI redacts HTTP(S) URLs and resolved header/cookie values from displayed - FFmpeg output. +- The CLI redacts HTTP(S) URLs and cookie values from displayed FFmpeg output and error lines. Header values may appear if FFmpeg echoes them. - Progress state stores item status/output and an identity key, not raw manifest URLs or credentials. diff --git a/site-docs/community.md b/website/community.md similarity index 87% rename from site-docs/community.md rename to website/community.md index 46602a5..9ac14f2 100644 --- a/site-docs/community.md +++ b/website/community.md @@ -18,4 +18,4 @@ - Repository: - Issues: -Public documentation lives in `site-docs/`. Repository-internal guidance is not published. +Public documentation lives in `website/`. Repository-internal guidance is not published. diff --git a/site-docs/getting-started.md b/website/getting-started.md similarity index 93% rename from site-docs/getting-started.md rename to website/getting-started.md index 81bd938..fb4ddcc 100644 --- a/site-docs/getting-started.md +++ b/website/getting-started.md @@ -2,9 +2,14 @@ ## Prerequisites +**Docker Compose:** + +- Docker Engine and Docker Compose + +**Direct local use:** + - Python 3.11+ - FFmpeg installed and available on `PATH` -- Docker Engine and Docker Compose (optional, for container use) ## Docker Compose diff --git a/site-docs/index.md b/website/index.md similarity index 83% rename from site-docs/index.md rename to website/index.md index a93db5d..8cd8212 100644 --- a/site-docs/index.md +++ b/website/index.md @@ -1,10 +1,12 @@ # ffmpegfyMe +**Download HTTP/HLS streams as MP4 or MKV — no transcoding.** + **ffmpegfyMe** is an interactive or batch Python wrapper around [FFmpeg](https://ffmpeg.org/) for downloading HTTP and HTTPS media streams, commonly HLS playlists, and saving them into local media files. -## What it does +## Implemented capabilities - Downloads HTTP/HTTPS media streams, commonly HLS (`.m3u8`) playlists. - Copies the stream into an `.mp4` or `.mkv` container using FFmpeg stream copy @@ -35,3 +37,7 @@ An `.mp4` or `.mkv` output is required; unsupported codec or container combinati fail and are reported per item. See [Troubleshooting](troubleshooting.md) for common failures. + +## Implemented today + +Feature descriptions across this site and the README describe implemented behavior. Planned work is tracked in GitHub Issues and may not be available yet. diff --git a/site-docs/manifest.md b/website/manifest.md similarity index 100% rename from site-docs/manifest.md rename to website/manifest.md diff --git a/site-docs/reliability.md b/website/reliability.md similarity index 89% rename from site-docs/reliability.md rename to website/reliability.md index 1ce421f..8c87040 100644 --- a/site-docs/reliability.md +++ b/website/reliability.md @@ -10,14 +10,14 @@ Inactivity timeouts and other FFmpeg download failures use this retry policy. Au - Default inactivity timeout: 60 seconds. - Any FFmpeg output resets the timer. -- Set with `FFMPEG_INACTIVITY_TIMEOUT`. +- Set `FFMPEG_INACTIVITY_TIMEOUT` in `.env` for Compose, or export it for direct local runs. - Must be a finite positive number. Invalid, non-finite, zero, and negative values produce a clear error before batch processing. ## No total-download cap -There is no total-download time limit by default. +No total-download timeout is implemented. ## Progress and state diff --git a/site-docs/stylesheets/catppuccin.css b/website/stylesheets/catppuccin.css similarity index 90% rename from site-docs/stylesheets/catppuccin.css rename to website/stylesheets/catppuccin.css index 20c823d..8d34a10 100644 --- a/site-docs/stylesheets/catppuccin.css +++ b/website/stylesheets/catppuccin.css @@ -11,7 +11,7 @@ --md-default-bg-color--lighter: #1e2030; --md-default-bg-color--lightest: #181926; --md-typeset-color: #cad3f5; - --md-typeset-a-color: #8aadf4; + --md-typeset-a-color: #91d7e3; --md-code-fg-color: #cad3f5; --md-code-bg-color: #1e2030; @@ -32,13 +32,13 @@ --md-code-hl-string-color: #a6da95; --md-code-hl-variable-color: #ee99a0; - --md-primary-fg-color: #8aadf4; + --md-primary-fg-color: #91d7e3; --md-primary-bg-color: #24273a; --md-primary-bg-color--light: #24273a; - --md-accent-fg-color: #8aadf4; + --md-accent-fg-color: #91d7e3; --md-accent-bg-color: #24273a; --md-accent-bg-color--light: #24273a; - --md-accent-fg-color--transparent: rgba(138, 173, 244, 0.1); + --md-accent-fg-color--transparent: rgba(145, 215, 227, 0.1); --md-admonition-bg-color: #363a4f; --md-admonition-fg-color: #cad3f5; diff --git a/site-docs/troubleshooting.md b/website/troubleshooting.md similarity index 89% rename from site-docs/troubleshooting.md rename to website/troubleshooting.md index 9deb021..2d0fad8 100644 --- a/site-docs/troubleshooting.md +++ b/website/troubleshooting.md @@ -26,7 +26,7 @@ Manual mode asks before overwrite. Automatic mode records a conflict/failure and Default is 60 seconds. Any FFmpeg output resets it. -Set `FFMPEG_INACTIVITY_TIMEOUT` to a finite positive number. +Set `FFMPEG_INACTIVITY_TIMEOUT` to a finite positive number in `.env` for Compose, or export it for direct local runs. ## Unsupported codecs/container combinations diff --git a/zensical.toml b/zensical.toml index 0bf7998..bc24a63 100644 --- a/zensical.toml +++ b/zensical.toml @@ -1,8 +1,8 @@ [project] site_name = "ffmpegfyMe" site_url = "https://fnayou.github.io/ffmpegfyMe/" -site_description = "Interactive or batch Python wrapper around FFmpeg for downloading HTTP and HTTPS media streams, commonly HLS playlists, and copying them into MP4 or MKV containers." -docs_dir = "site-docs" +site_description = "Interactive or batch Python CLI that downloads HTTP and HTTPS media streams, commonly HLS playlists, via FFmpeg stream copy into MP4 or MKV outputs with retries and progress tracking." +docs_dir = "website" site_dir = "site" repo_url = "https://github.com/fnayou/ffmpegfyMe" @@ -20,6 +20,8 @@ extra_css = ["stylesheets/catppuccin.css"] [project.theme] language = "en" +logo = "assets/logo.svg" +favicon = "assets/favicon.svg" palette = [ { scheme = "macchiato", primary = "custom", accent = "custom" }, ]