From dcff3ad08583b822b86d6da86e83b6c8c3810af9 Mon Sep 17 00:00:00 2001 From: Mika Cohen Date: Tue, 7 Jul 2026 08:02:15 -0600 Subject: [PATCH 01/70] Add Serena project memories --- .gitignore | 2 ++ .serena/memories/conventions.md | 8 +++++++ .serena/memories/core.md | 8 +++++++ .serena/memories/memory_maintenance.md | 33 ++++++++++++++++++++++++++ .serena/memories/suggested_commands.md | 9 +++++++ .serena/memories/task_completion.md | 6 +++++ .serena/memories/tech_stack.md | 8 +++++++ 7 files changed, 74 insertions(+) create mode 100644 .serena/memories/conventions.md create mode 100644 .serena/memories/core.md create mode 100644 .serena/memories/memory_maintenance.md create mode 100644 .serena/memories/suggested_commands.md create mode 100644 .serena/memories/task_completion.md create mode 100644 .serena/memories/tech_stack.md diff --git a/.gitignore b/.gitignore index b7a7ba40..b134fb86 100644 --- a/.gitignore +++ b/.gitignore @@ -46,3 +46,5 @@ config/dev.overrides.exs .expert/ .envrc + +.serena/cache/ diff --git a/.serena/memories/conventions.md b/.serena/memories/conventions.md new file mode 100644 index 00000000..6e8f8a22 --- /dev/null +++ b/.serena/memories/conventions.md @@ -0,0 +1,8 @@ +# Conventions + +- Use existing state-machine helpers for video state transitions when available. +- Treat `:encoded` videos as protected during sync; do not reset encoded state or discard chosen VMAF/original-size data. +- `Reencodarr.AbAv1.OutputParser` is the shared parser for ab-av1 output; keep encode and CRF-search parsing aligned there. +- `Reencodarr.Rules.build_args/4` is the central encode/CRF-search argument builder. +- Service clients live under `lib/reencodarr/services/` and follow existing CarReq retry/fuse patterns. +- Paginated list LiveViews use URL params plus `ReencodarrWeb.Live.FlopList`; list APIs return `{items, %Flop.Meta{}}`. \ No newline at end of file diff --git a/.serena/memories/core.md b/.serena/memories/core.md new file mode 100644 index 00000000..7200d02d --- /dev/null +++ b/.serena/memories/core.md @@ -0,0 +1,8 @@ +# Core + +- Phoenix app for Reencodarr media transcoding; web modules under `lib/reencodarr_web`, domain modules under `lib/reencodarr`. +- SQLite via `Reencodarr.Repo`; keep DB tuning centralized in `config/config.exs`. +- Broadway/local worker supervisors are disabled in tests via `config :reencodarr, env: :test` and `Application.worker_children/0`. +- Existing app invariants and live-debug commands live in `AGENTS.md`; prefer repo evidence, logs, diagnostics, and tests over guessing. + +Related memories: commands in `mem:suggested_commands`, done gates in `mem:task_completion`, style in `mem:conventions`, stack in `mem:tech_stack`. \ No newline at end of file diff --git a/.serena/memories/memory_maintenance.md b/.serena/memories/memory_maintenance.md new file mode 100644 index 00000000..6f84514d --- /dev/null +++ b/.serena/memories/memory_maintenance.md @@ -0,0 +1,33 @@ +# Memory Maintenance + +## Discovery Model + +- Core principle: progressive discovery through references, building a graph of memories. +- Initially, agents are provided with the list of all memories (names only). +- Agents should read `mem:core` as the top-level entry point (graph root). + This memory should contain references to other memories covering major project domains. + The referenced memories shall, in turn, shall contain references to even more specific memories, and so on. + The depth of the graph shall depend on the project complexity. +- Use topics/folders to group related memories in order to make the content structure explicit. + Folders can mirror project structure (e.g. modules like frontend/backend) or topics like debugging, architecture, etc. +- Memory references must use a mem: prefix inside backticks, e.g. `mem:frontend/core`. + The surrounding text should clearly indicate when to read the memory/which content to expect. + The text should provide more precise guidance than the memory name alone, + i.e. avoid a reference like "frontend debugging: `mem:frontend/debugging` and instead make clear which aspects of frontend debugging are covered. +- Memories themselves should not contain information about when to read them; this is the responsibility of the referring memory. + +## Style + +Dense agent notes, not prose docs. Prefer invariants, terse bullets. +Avoid obvious context, rationale, and examples unless they prevent likely mistakes. +Keep guidance durable and generalizable, not task-local. + +## Add/update threshold + +Add or update memories only with stable, non-obvious project conventions that avoid complex rediscovery in the future. +Do not add: quick-read facts; generic language/framework knowledge; one-off task notes; volatile line-level details; behavior likely to change soon. + +## Maintenance Actions + +- Renaming memories: References are updated automatically if handled via Serena's memory rename tool. +- Checking for stale memories (e.g. after deletion): Call `serena memories check` for a report. \ No newline at end of file diff --git a/.serena/memories/suggested_commands.md b/.serena/memories/suggested_commands.md new file mode 100644 index 00000000..2aa67f2a --- /dev/null +++ b/.serena/memories/suggested_commands.md @@ -0,0 +1,9 @@ +# Suggested Commands + +- `nix develop -c mix setup` installs deps, creates/migrates DB, and builds assets. +- `nix develop -c mix test` runs the full test suite. +- `nix develop -c mix test path/to/test.exs` runs focused tests. +- `nix develop -c mix compile --warnings-as-errors` catches compile regressions. +- `nix develop -c mix credo --strict` runs lint. +- `nix develop -c mix format` formats; `nix develop -c mix format --check-formatted` verifies formatting. +- `bin/rpc 'Reencodarr.Diagnostics.status()'` and sibling diagnostics helpers inspect the running system. \ No newline at end of file diff --git a/.serena/memories/task_completion.md b/.serena/memories/task_completion.md new file mode 100644 index 00000000..389c47a5 --- /dev/null +++ b/.serena/memories/task_completion.md @@ -0,0 +1,6 @@ +# Task Completion + +- Prefer focused tests while iterating, then run broader gates when practical. +- Standard finish gates: `nix develop -c mix test`, `nix develop -c mix compile --warnings-as-errors`, `nix develop -c mix credo --strict`, and `nix develop -c mix format --check-formatted`. +- Run `nix develop -c mix format` before committing Elixir changes if formatting may have changed. +- For live runtime/debug claims, verify with `bin/rpc` diagnostics rather than inferring from source. \ No newline at end of file diff --git a/.serena/memories/tech_stack.md b/.serena/memories/tech_stack.md new file mode 100644 index 00000000..2e4a39be --- /dev/null +++ b/.serena/memories/tech_stack.md @@ -0,0 +1,8 @@ +# Tech Stack + +- Elixir `>= 1.20.0 and < 1.21.0` Phoenix app. +- Phoenix `~> 1.8.8`, LiveView `~> 1.2.5`, Bandit `~> 1.12`. +- SQLite through `ecto_sqlite3`/`exqlite`; Ecto SQL `~> 3.14`. +- Uses Nix dev shell for repo commands: `nix develop -c ...`. +- Assets use Tailwind package `~> 0.5.1` and esbuild `~> 0.10` via Mix aliases. +- Test helpers include ExUnit, ConnCase/DataCase, StreamData, Floki/LazyHTML, and meck where existing tests mock external commands. \ No newline at end of file From 85bd085563a03d2f8c54bcd788039256d6524b19 Mon Sep 17 00:00:00 2001 From: Mika Cohen Date: Tue, 7 Jul 2026 08:03:44 -0600 Subject: [PATCH 02/70] Add ab-av1 worker websocket handshake --- lib/reencodarr_web/channels/worker_channel.ex | 34 +++++++++++++++ lib/reencodarr_web/channels/worker_socket.ex | 25 +++++++++++ lib/reencodarr_web/endpoint.ex | 4 ++ .../channels/worker_channel_test.exs | 41 +++++++++++++++++++ test/support/channel_case.ex | 20 +++++++++ 5 files changed, 124 insertions(+) create mode 100644 lib/reencodarr_web/channels/worker_channel.ex create mode 100644 lib/reencodarr_web/channels/worker_socket.ex create mode 100644 test/reencodarr_web/channels/worker_channel_test.exs create mode 100644 test/support/channel_case.ex diff --git a/lib/reencodarr_web/channels/worker_channel.ex b/lib/reencodarr_web/channels/worker_channel.ex new file mode 100644 index 00000000..317dae24 --- /dev/null +++ b/lib/reencodarr_web/channels/worker_channel.ex @@ -0,0 +1,34 @@ +defmodule ReencodarrWeb.WorkerChannel do + @moduledoc """ + Phoenix channel for long-lived ab-av1 CRF-search workers. + """ + + use ReencodarrWeb, :channel + + @impl true + def join("workers:crf_search", _payload, %{assigns: %{worker_id: worker_id}} = socket) do + {:ok, %{worker_id: worker_id}, socket} + end + + def join(_topic, _payload, socket), do: {:error, %{reason: "unauthorized"}, socket} + + @impl true + def handle_in("announce", payload, socket) do + case payload do + %{"worker_id" => worker_id, "version" => version, "capabilities" => capabilities} + when is_binary(worker_id) and is_binary(version) and is_map(capabilities) -> + socket = + socket + |> assign(:client_worker_id, worker_id) + |> assign(:client_version, version) + |> assign(:capabilities, capabilities) + + {:reply, {:ok, %{accepted: true}}, socket} + + _invalid -> + {:reply, {:error, %{reason: "invalid_announcement"}}, socket} + end + end + + def handle_in("pull_work", _payload, socket), do: {:reply, {:ok, %{status: "no_work"}}, socket} +end diff --git a/lib/reencodarr_web/channels/worker_socket.ex b/lib/reencodarr_web/channels/worker_socket.ex new file mode 100644 index 00000000..e2304c4c --- /dev/null +++ b/lib/reencodarr_web/channels/worker_socket.ex @@ -0,0 +1,25 @@ +defmodule ReencodarrWeb.WorkerSocket do + @moduledoc """ + Authenticated websocket socket for ab-av1 worker clients. + """ + + use Phoenix.Socket + + channel "workers:crf_search", ReencodarrWeb.WorkerChannel + + @impl true + def connect(%{"token" => token}, socket, _connect_info) do + case Application.fetch_env(:reencodarr, :worker_token) do + {:ok, ^token} -> {:ok, assign(socket, :worker_id, worker_id())} + _ -> :error + end + end + + def connect(_params, _socket, _connect_info), do: :error + + @impl true + def id(%{assigns: %{worker_id: worker_id}}), do: "worker_socket:#{worker_id}" + def id(_socket), do: nil + + defp worker_id, do: "worker-" <> Base.encode16(:crypto.strong_rand_bytes(8), case: :lower) +end diff --git a/lib/reencodarr_web/endpoint.ex b/lib/reencodarr_web/endpoint.ex index d2c5d1b7..cdb46115 100644 --- a/lib/reencodarr_web/endpoint.ex +++ b/lib/reencodarr_web/endpoint.ex @@ -18,6 +18,10 @@ defmodule ReencodarrWeb.Endpoint do # Socket for embedded LiveViews (like Broadway Dashboard in iframe) socket "/embed/live", Phoenix.LiveView.Socket, websocket: true, longpoll: true + socket "/workers/socket", ReencodarrWeb.WorkerSocket, + websocket: true, + longpoll: false + # Serve at "/" the static files from "priv/static" directory. # # You should set gzip to true if you are running phx.digest diff --git a/test/reencodarr_web/channels/worker_channel_test.exs b/test/reencodarr_web/channels/worker_channel_test.exs new file mode 100644 index 00000000..53dc70b1 --- /dev/null +++ b/test/reencodarr_web/channels/worker_channel_test.exs @@ -0,0 +1,41 @@ +defmodule ReencodarrWeb.WorkerChannelTest do + use ReencodarrWeb.ChannelCase, async: true + + alias ReencodarrWeb.WorkerSocket + + describe "ab-av1 worker websocket" do + test "authenticates, announces capabilities, and returns no work" do + token = "test-worker-token" + Application.put_env(:reencodarr, :worker_token, token) + + assert {:ok, socket} = connect(WorkerSocket, %{"token" => token}) + assert socket.assigns.worker_id =~ "worker-" + + assert {:ok, %{worker_id: worker_id}, socket} = + subscribe_and_join(socket, "workers:crf_search") + + assert worker_id == socket.assigns.worker_id + + assert_reply push(socket, "announce", %{ + "worker_id" => "abav1-dev", + "version" => "0.10.0", + "capabilities" => %{"crf_search" => true} + }), + :ok, + %{accepted: true} + + assert_reply push(socket, "pull_work", %{}), :ok, %{status: "no_work"} + after + Application.delete_env(:reencodarr, :worker_token) + end + + test "rejects missing or invalid tokens" do + Application.put_env(:reencodarr, :worker_token, "test-worker-token") + + assert :error = connect(WorkerSocket, %{}) + assert :error = connect(WorkerSocket, %{"token" => "wrong"}) + after + Application.delete_env(:reencodarr, :worker_token) + end + end +end diff --git a/test/support/channel_case.ex b/test/support/channel_case.ex new file mode 100644 index 00000000..ed2cc9dd --- /dev/null +++ b/test/support/channel_case.ex @@ -0,0 +1,20 @@ +defmodule ReencodarrWeb.ChannelCase do + @moduledoc false + + use ExUnit.CaseTemplate + + using do + quote do + @endpoint ReencodarrWeb.Endpoint + + import Phoenix.ChannelTest + + import ReencodarrWeb.ChannelCase + end + end + + setup tags do + Reencodarr.DataCase.setup_sandbox(tags) + :ok + end +end From b0a911f48beb3461550b920be8e42ab0f695ad84 Mon Sep 17 00:00:00 2001 From: Mika Cohen Date: Tue, 7 Jul 2026 08:08:11 -0600 Subject: [PATCH 03/70] Extract ab-av1 worker protocol helpers --- lib/reencodarr/ab_av1/worker_protocol.ex | 30 +++++++++++++++++++ lib/reencodarr_web/channels/worker_channel.ex | 22 ++++++++------ lib/reencodarr_web/channels/worker_socket.ex | 4 ++- 3 files changed, 46 insertions(+), 10 deletions(-) create mode 100644 lib/reencodarr/ab_av1/worker_protocol.ex diff --git a/lib/reencodarr/ab_av1/worker_protocol.ex b/lib/reencodarr/ab_av1/worker_protocol.ex new file mode 100644 index 00000000..ce6eadce --- /dev/null +++ b/lib/reencodarr/ab_av1/worker_protocol.ex @@ -0,0 +1,30 @@ +defmodule Reencodarr.AbAv1.WorkerProtocol do + @moduledoc """ + Server-side helpers for the ab-av1 worker websocket protocol. + """ + + @crf_search_topic "workers:crf_search" + + def crf_search_topic, do: @crf_search_topic + + def valid_topic?(@crf_search_topic), do: true + def valid_topic?(_topic), do: false + + def parse_announcement(%{ + "worker_id" => worker_id, + "version" => version, + "capabilities" => capabilities + }) + when is_binary(worker_id) and is_binary(version) and is_map(capabilities) do + {:ok, %{worker_id: worker_id, version: version, capabilities: capabilities}} + end + + def parse_announcement(_payload), do: {:error, :invalid_announcement} + + def accepted, do: %{accepted: true} + + def no_work, do: %{status: "no_work"} + + def error(:invalid_announcement), do: %{reason: "invalid_announcement"} + def error(:unauthorized), do: %{reason: "unauthorized"} +end diff --git a/lib/reencodarr_web/channels/worker_channel.ex b/lib/reencodarr_web/channels/worker_channel.ex index 317dae24..4e5f3c44 100644 --- a/lib/reencodarr_web/channels/worker_channel.ex +++ b/lib/reencodarr_web/channels/worker_channel.ex @@ -5,30 +5,34 @@ defmodule ReencodarrWeb.WorkerChannel do use ReencodarrWeb, :channel + alias Reencodarr.AbAv1.WorkerProtocol + + @crf_search_topic WorkerProtocol.crf_search_topic() + @impl true - def join("workers:crf_search", _payload, %{assigns: %{worker_id: worker_id}} = socket) do + def join(@crf_search_topic, _payload, %{assigns: %{worker_id: worker_id}} = socket) do {:ok, %{worker_id: worker_id}, socket} end - def join(_topic, _payload, socket), do: {:error, %{reason: "unauthorized"}, socket} + def join(_topic, _payload, socket), do: {:error, WorkerProtocol.error(:unauthorized), socket} @impl true def handle_in("announce", payload, socket) do - case payload do - %{"worker_id" => worker_id, "version" => version, "capabilities" => capabilities} - when is_binary(worker_id) and is_binary(version) and is_map(capabilities) -> + case WorkerProtocol.parse_announcement(payload) do + {:ok, %{worker_id: worker_id, version: version, capabilities: capabilities}} -> socket = socket |> assign(:client_worker_id, worker_id) |> assign(:client_version, version) |> assign(:capabilities, capabilities) - {:reply, {:ok, %{accepted: true}}, socket} + {:reply, {:ok, WorkerProtocol.accepted()}, socket} - _invalid -> - {:reply, {:error, %{reason: "invalid_announcement"}}, socket} + {:error, reason} -> + {:reply, {:error, WorkerProtocol.error(reason)}, socket} end end - def handle_in("pull_work", _payload, socket), do: {:reply, {:ok, %{status: "no_work"}}, socket} + def handle_in("pull_work", _payload, socket), + do: {:reply, {:ok, WorkerProtocol.no_work()}, socket} end diff --git a/lib/reencodarr_web/channels/worker_socket.ex b/lib/reencodarr_web/channels/worker_socket.ex index e2304c4c..94e64f43 100644 --- a/lib/reencodarr_web/channels/worker_socket.ex +++ b/lib/reencodarr_web/channels/worker_socket.ex @@ -5,7 +5,9 @@ defmodule ReencodarrWeb.WorkerSocket do use Phoenix.Socket - channel "workers:crf_search", ReencodarrWeb.WorkerChannel + alias Reencodarr.AbAv1.WorkerProtocol + + channel WorkerProtocol.crf_search_topic(), ReencodarrWeb.WorkerChannel @impl true def connect(%{"token" => token}, socket, _connect_info) do From 0c5738a13bb3419d618ba4e6b8613fb56c523ac1 Mon Sep 17 00:00:00 2001 From: Mika Cohen Date: Tue, 7 Jul 2026 08:11:06 -0600 Subject: [PATCH 04/70] Document ab-av1 worker websocket contract --- config/runtime.exs | 4 +++ docs/ab_av1_worker_websocket.md | 44 +++++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+) create mode 100644 docs/ab_av1_worker_websocket.md diff --git a/config/runtime.exs b/config/runtime.exs index a59b4375..ad9093c4 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -20,6 +20,10 @@ if System.get_env("PHX_SERVER") do config :reencodarr, ReencodarrWeb.Endpoint, server: true end +if worker_token = System.get_env("REENCODARR_WORKER_TOKEN") do + config :reencodarr, :worker_token, worker_token +end + if config_env() == :prod do # Parse CHECK_ORIGIN env var: "false" disables, otherwise comma-separated origin list parse_check_origin = fn diff --git a/docs/ab_av1_worker_websocket.md b/docs/ab_av1_worker_websocket.md new file mode 100644 index 00000000..7c42620f --- /dev/null +++ b/docs/ab_av1_worker_websocket.md @@ -0,0 +1,44 @@ +# ab-av1 Worker Websocket + +First REENC-133 server contract for CRF-search workers. + +## Connection + +- Websocket URL: `/workers/socket/websocket?token=` +- Runtime token env: `REENCODARR_WORKER_TOKEN` +- Phoenix channel topic: `workers:crf_search` + +## Events + +After joining `workers:crf_search`, send: + +```json +{ + "event": "announce", + "payload": { + "worker_id": "abav1-dev", + "version": "0.10.0", + "capabilities": {"crf_search": true} + } +} +``` + +Expected reply payload: + +```json +{"accepted": true} +``` + +Then workers can ask for work: + +```json +{"event": "pull_work", "payload": {}} +``` + +Current first-pass reply when no distributed job is available: + +```json +{"status": "no_work"} +``` + +File transfer, chunk checksums, progress, result reporting, and requeue/failure handling are still pending REENC-133 follow-up slices. From 2a233789356dd9fc88d92e3a599de3d0d12a16e0 Mon Sep 17 00:00:00 2001 From: Mika Cohen Date: Tue, 7 Jul 2026 08:22:11 -0600 Subject: [PATCH 05/70] Clarify worker socket uses main Phoenix endpoint --- docs/ab_av1_worker_websocket.md | 1 + lib/reencodarr_web/endpoint.ex | 2 ++ 2 files changed, 3 insertions(+) diff --git a/docs/ab_av1_worker_websocket.md b/docs/ab_av1_worker_websocket.md index 7c42620f..0e61f50d 100644 --- a/docs/ab_av1_worker_websocket.md +++ b/docs/ab_av1_worker_websocket.md @@ -5,6 +5,7 @@ First REENC-133 server contract for CRF-search workers. ## Connection - Websocket URL: `/workers/socket/websocket?token=` +- This is served by the main Phoenix app endpoint on the same host/port as the dashboard and API. It is not a separate service port. - Runtime token env: `REENCODARR_WORKER_TOKEN` - Phoenix channel topic: `workers:crf_search` diff --git a/lib/reencodarr_web/endpoint.ex b/lib/reencodarr_web/endpoint.ex index cdb46115..aa9586dc 100644 --- a/lib/reencodarr_web/endpoint.ex +++ b/lib/reencodarr_web/endpoint.ex @@ -18,6 +18,8 @@ defmodule ReencodarrWeb.Endpoint do # Socket for embedded LiveViews (like Broadway Dashboard in iframe) socket "/embed/live", Phoenix.LiveView.Socket, websocket: true, longpoll: true + # ab-av1 workers connect through the main Phoenix endpoint on the same + # host/port as the rest of the app. socket "/workers/socket", ReencodarrWeb.WorkerSocket, websocket: true, longpoll: false From e7438bb53733994e1dccf299b6648223a9296f97 Mon Sep 17 00:00:00 2001 From: Mika Cohen Date: Tue, 7 Jul 2026 08:39:06 -0600 Subject: [PATCH 06/70] Track websocket worker sessions --- lib/reencodarr/ab_av1/worker_protocol.ex | 34 ++++- lib/reencodarr/ab_av1/worker_sessions.ex | 138 ++++++++++++++++++ lib/reencodarr/application.ex | 1 + lib/reencodarr_web/channels/worker_channel.ex | 54 +++++-- .../channels/worker_channel_test.exs | 71 ++++++++- 5 files changed, 284 insertions(+), 14 deletions(-) create mode 100644 lib/reencodarr/ab_av1/worker_sessions.ex diff --git a/lib/reencodarr/ab_av1/worker_protocol.ex b/lib/reencodarr/ab_av1/worker_protocol.ex index ce6eadce..1ba4a233 100644 --- a/lib/reencodarr/ab_av1/worker_protocol.ex +++ b/lib/reencodarr/ab_av1/worker_protocol.ex @@ -4,27 +4,55 @@ defmodule Reencodarr.AbAv1.WorkerProtocol do """ @crf_search_topic "workers:crf_search" + @supported_protocol_versions [1] def crf_search_topic, do: @crf_search_topic + def supported_protocol_versions, do: @supported_protocol_versions + def valid_topic?(@crf_search_topic), do: true def valid_topic?(_topic), do: false + def supported_protocol_version?(protocol_version), + do: protocol_version in @supported_protocol_versions + def parse_announcement(%{ "worker_id" => worker_id, + "protocol_version" => protocol_version, "version" => version, "capabilities" => capabilities }) - when is_binary(worker_id) and is_binary(version) and is_map(capabilities) do - {:ok, %{worker_id: worker_id, version: version, capabilities: capabilities}} + when is_binary(worker_id) and is_integer(protocol_version) and is_binary(version) and + is_map(capabilities) do + {:ok, + %{ + worker_id: worker_id, + protocol_version: protocol_version, + version: version, + capabilities: capabilities + }} end def parse_announcement(_payload), do: {:error, :invalid_announcement} - def accepted, do: %{accepted: true} + def accepted(protocol_version), do: %{accepted: true, protocol_version: protocol_version} def no_work, do: %{status: "no_work"} + def heartbeat_ack(last_seen_at), + do: %{accepted: true, last_seen_at: DateTime.to_iso8601(last_seen_at)} + + def error(:duplicate_worker_id), do: %{reason: "duplicate_worker_id"} + def error(:invalid_announcement), do: %{reason: "invalid_announcement"} + + def error(:unsupported_protocol_version) do + %{ + reason: "unsupported_protocol_version", + supported_protocol_versions: @supported_protocol_versions + } + end + + def error(:unknown_worker_session), do: %{reason: "unknown_worker_session"} def error(:unauthorized), do: %{reason: "unauthorized"} end diff --git a/lib/reencodarr/ab_av1/worker_sessions.ex b/lib/reencodarr/ab_av1/worker_sessions.ex new file mode 100644 index 00000000..1b417bdd --- /dev/null +++ b/lib/reencodarr/ab_av1/worker_sessions.ex @@ -0,0 +1,138 @@ +defmodule Reencodarr.AbAv1.WorkerSessions do + @moduledoc """ + Tracks connected ab-av1 worker websocket sessions. + """ + + use GenServer + + @type session :: %{ + server_worker_id: String.t(), + client_worker_id: String.t(), + version: String.t(), + protocol_version: pos_integer(), + capabilities: map(), + connected_at: DateTime.t(), + last_seen_at: DateTime.t() + } + + def start_link(opts) do + GenServer.start_link(__MODULE__, opts, name: __MODULE__) + end + + def register(attrs) do + GenServer.call(__MODULE__, {:register, attrs}) + end + + def touch(server_worker_id) do + GenServer.call(__MODULE__, {:touch, server_worker_id}) + end + + def unregister(server_worker_id) do + GenServer.call(__MODULE__, {:unregister, server_worker_id}) + end + + def list do + GenServer.call(__MODULE__, :list) + end + + def reset do + GenServer.call(__MODULE__, :reset) + end + + @impl GenServer + def init(_opts) do + {:ok, %{by_server: %{}, by_client: %{}}} + end + + @impl GenServer + def handle_call({:register, attrs}, _from, state) do + now = now() + server_worker_id = Map.fetch!(attrs, :server_worker_id) + client_worker_id = Map.fetch!(attrs, :client_worker_id) + + case Map.get(state.by_client, client_worker_id) do + nil -> + session = build_session(attrs, now) + {:reply, {:ok, session}, put_session(state, session)} + + ^server_worker_id -> + session = + attrs + |> build_session(now) + |> Map.put(:connected_at, state.by_server[server_worker_id].connected_at) + + {:reply, {:ok, session}, put_session(state, session)} + + _other_server_worker_id -> + {:reply, {:error, :duplicate_worker_id}, state} + end + end + + def handle_call({:touch, server_worker_id}, _from, state) do + case Map.fetch(state.by_server, server_worker_id) do + {:ok, session} -> + updated_session = %{session | last_seen_at: now()} + {:reply, {:ok, updated_session}, put_session(state, updated_session)} + + :error -> + {:reply, {:error, :unknown_worker_session}, state} + end + end + + def handle_call({:unregister, server_worker_id}, _from, state) do + {:reply, :ok, drop_session(state, server_worker_id)} + end + + def handle_call(:list, _from, state) do + sessions = + state.by_server + |> Map.values() + |> Enum.sort_by(& &1.client_worker_id) + + {:reply, sessions, state} + end + + def handle_call(:reset, _from, _state) do + {:reply, :ok, %{by_server: %{}, by_client: %{}}} + end + + defp build_session(attrs, now) do + %{ + server_worker_id: Map.fetch!(attrs, :server_worker_id), + client_worker_id: Map.fetch!(attrs, :client_worker_id), + version: Map.fetch!(attrs, :version), + protocol_version: Map.fetch!(attrs, :protocol_version), + capabilities: Map.fetch!(attrs, :capabilities), + connected_at: now, + last_seen_at: now + } + end + + defp put_session(state, session) do + state + |> drop_session(session.server_worker_id) + |> then(fn state -> + %{ + by_server: Map.put(state.by_server, session.server_worker_id, session), + by_client: Map.put(state.by_client, session.client_worker_id, session.server_worker_id) + } + end) + end + + defp drop_session(state, server_worker_id) do + case Map.pop(state.by_server, server_worker_id) do + {nil, by_server} -> + %{state | by_server: by_server} + + {session, by_server} -> + %{ + by_server: by_server, + by_client: Map.delete(state.by_client, session.client_worker_id) + } + end + end + + defp now do + DateTime.utc_now() |> DateTime.truncate(:second) + end +end diff --git a/lib/reencodarr/application.ex b/lib/reencodarr/application.ex index eade7a9f..329cf63f 100644 --- a/lib/reencodarr/application.ex +++ b/lib/reencodarr/application.ex @@ -109,6 +109,7 @@ defmodule Reencodarr.Application do {Phoenix.PubSub, name: Reencodarr.PubSub}, # Start the Finch HTTP client for outbound API requests. {Finch, name: Reencodarr.Finch}, + Reencodarr.AbAv1.WorkerSessions, # Start to serve requests, typically the last entry ReencodarrWeb.Endpoint, # DynamicSupervisor for port-holder processes (AbAv1.Encoder, AbAv1.CrfSearcher). diff --git a/lib/reencodarr_web/channels/worker_channel.ex b/lib/reencodarr_web/channels/worker_channel.ex index 4e5f3c44..6c5c9109 100644 --- a/lib/reencodarr_web/channels/worker_channel.ex +++ b/lib/reencodarr_web/channels/worker_channel.ex @@ -5,7 +5,7 @@ defmodule ReencodarrWeb.WorkerChannel do use ReencodarrWeb, :channel - alias Reencodarr.AbAv1.WorkerProtocol + alias Reencodarr.AbAv1.{WorkerProtocol, WorkerSessions} @crf_search_topic WorkerProtocol.crf_search_topic() @@ -17,16 +17,44 @@ defmodule ReencodarrWeb.WorkerChannel do def join(_topic, _payload, socket), do: {:error, WorkerProtocol.error(:unauthorized), socket} @impl true - def handle_in("announce", payload, socket) do - case WorkerProtocol.parse_announcement(payload) do - {:ok, %{worker_id: worker_id, version: version, capabilities: capabilities}} -> - socket = - socket - |> assign(:client_worker_id, worker_id) - |> assign(:client_version, version) - |> assign(:capabilities, capabilities) + def handle_in("announce", payload, %{assigns: %{worker_id: server_worker_id}} = socket) do + with {:ok, + %{ + worker_id: client_worker_id, + protocol_version: protocol_version, + version: version, + capabilities: capabilities + }} <- WorkerProtocol.parse_announcement(payload), + true <- WorkerProtocol.supported_protocol_version?(protocol_version), + {:ok, _session} <- + WorkerSessions.register(%{ + server_worker_id: server_worker_id, + client_worker_id: client_worker_id, + protocol_version: protocol_version, + version: version, + capabilities: capabilities + }) do + socket = + socket + |> assign(:client_worker_id, client_worker_id) + |> assign(:client_version, version) + |> assign(:protocol_version, protocol_version) + |> assign(:capabilities, capabilities) - {:reply, {:ok, WorkerProtocol.accepted()}, socket} + {:reply, {:ok, WorkerProtocol.accepted(protocol_version)}, socket} + else + false -> + {:reply, {:error, WorkerProtocol.error(:unsupported_protocol_version)}, socket} + + {:error, reason} -> + {:reply, {:error, WorkerProtocol.error(reason)}, socket} + end + end + + def handle_in("heartbeat", _payload, %{assigns: %{worker_id: worker_id}} = socket) do + case WorkerSessions.touch(worker_id) do + {:ok, session} -> + {:reply, {:ok, WorkerProtocol.heartbeat_ack(session.last_seen_at)}, socket} {:error, reason} -> {:reply, {:error, WorkerProtocol.error(reason)}, socket} @@ -35,4 +63,10 @@ defmodule ReencodarrWeb.WorkerChannel do def handle_in("pull_work", _payload, socket), do: {:reply, {:ok, WorkerProtocol.no_work()}, socket} + + @impl true + def terminate(_reason, %{assigns: %{worker_id: worker_id}}) do + WorkerSessions.unregister(worker_id) + :ok + end end diff --git a/test/reencodarr_web/channels/worker_channel_test.exs b/test/reencodarr_web/channels/worker_channel_test.exs index 53dc70b1..54389ae4 100644 --- a/test/reencodarr_web/channels/worker_channel_test.exs +++ b/test/reencodarr_web/channels/worker_channel_test.exs @@ -1,9 +1,15 @@ defmodule ReencodarrWeb.WorkerChannelTest do use ReencodarrWeb.ChannelCase, async: true + alias Reencodarr.AbAv1.WorkerSessions alias ReencodarrWeb.WorkerSocket describe "ab-av1 worker websocket" do + setup do + WorkerSessions.reset() + :ok + end + test "authenticates, announces capabilities, and returns no work" do token = "test-worker-token" Application.put_env(:reencodarr, :worker_token, token) @@ -18,17 +24,80 @@ defmodule ReencodarrWeb.WorkerChannelTest do assert_reply push(socket, "announce", %{ "worker_id" => "abav1-dev", + "protocol_version" => 1, "version" => "0.10.0", "capabilities" => %{"crf_search" => true} }), :ok, - %{accepted: true} + %{accepted: true, protocol_version: 1} assert_reply push(socket, "pull_work", %{}), :ok, %{status: "no_work"} after Application.delete_env(:reencodarr, :worker_token) end + test "rejects unsupported protocol versions" do + token = "test-worker-token" + Application.put_env(:reencodarr, :worker_token, token) + + assert {:ok, socket} = connect(WorkerSocket, %{"token" => token}) + assert {:ok, _join_payload, socket} = subscribe_and_join(socket, "workers:crf_search") + + assert_reply push(socket, "announce", %{ + "worker_id" => "abav1-dev", + "protocol_version" => 99, + "version" => "0.10.0", + "capabilities" => %{"crf_search" => true} + }), + :error, + %{reason: "unsupported_protocol_version", supported_protocol_versions: [1]} + after + Application.delete_env(:reencodarr, :worker_token) + end + + test "rejects duplicate worker ids until the first session disconnects" do + token = "test-worker-token" + Application.put_env(:reencodarr, :worker_token, token) + + assert {:ok, socket1} = connect(WorkerSocket, %{"token" => token}) + assert {:ok, _join_payload, socket1} = subscribe_and_join(socket1, "workers:crf_search") + + assert_reply push(socket1, "announce", %{ + "worker_id" => "abav1-dev", + "protocol_version" => 1, + "version" => "0.10.0", + "capabilities" => %{"crf_search" => true} + }), + :ok, + %{accepted: true, protocol_version: 1} + + assert {:ok, socket2} = connect(WorkerSocket, %{"token" => token}) + assert {:ok, _join_payload, socket2} = subscribe_and_join(socket2, "workers:crf_search") + + assert_reply push(socket2, "announce", %{ + "worker_id" => "abav1-dev", + "protocol_version" => 1, + "version" => "0.10.0", + "capabilities" => %{"crf_search" => true} + }), + :error, + %{reason: "duplicate_worker_id"} + + Process.unlink(socket1.channel_pid) + assert :ok = close(socket1) + + assert_reply push(socket2, "announce", %{ + "worker_id" => "abav1-dev", + "protocol_version" => 1, + "version" => "0.10.0", + "capabilities" => %{"crf_search" => true} + }), + :ok, + %{accepted: true, protocol_version: 1} + after + Application.delete_env(:reencodarr, :worker_token) + end + test "rejects missing or invalid tokens" do Application.put_env(:reencodarr, :worker_token, "test-worker-token") From 49581d6b70fcd60956495effa8c851a4ec8aa166 Mon Sep 17 00:00:00 2001 From: Mika Cohen Date: Tue, 7 Jul 2026 08:56:29 -0600 Subject: [PATCH 07/70] Show worker websocket details on dashboard --- lib/reencodarr_web/live/dashboard_live.ex | 108 ++++++++++++------ .../channels/worker_channel_test.exs | 79 +++++++++---- .../live/dashboard_v2_live_test.exs | 19 +++ 3 files changed, 149 insertions(+), 57 deletions(-) diff --git a/lib/reencodarr_web/live/dashboard_live.ex b/lib/reencodarr_web/live/dashboard_live.ex index 02e2b9c0..c7f92578 100644 --- a/lib/reencodarr_web/live/dashboard_live.ex +++ b/lib/reencodarr_web/live/dashboard_live.ex @@ -51,6 +51,8 @@ defmodule ReencodarrWeb.DashboardLive do sync_progress: 0, service_type: nil, page_title: nil, + worker_token: Application.get_env(:reencodarr, :worker_token), + worker_socket_url: worker_socket_url(), # New dashboard stats stats: Reencodarr.Media.get_default_stats(), stats_display: stats_display(Reencodarr.Media.get_default_stats()), @@ -986,49 +988,80 @@ defmodule ReencodarrWeb.DashboardLive do attr :syncing, :boolean, required: true attr :sync_progress, :integer, required: true attr :service_type, :atom, required: true + attr :worker_token, :string, default: nil + attr :worker_socket_url, :string, required: true defp sync_controls(assigns) do ~H""" -
-
-

Media Library Sync

- -
- - +
+
+
+

Media Library Sync

+ +
+ + +
+ + <%= if @syncing do %> +
+
+
+
+
+
+ Syncing {@service_type}... {@sync_progress}% +
+
+ <% end %>
- <%= if @syncing do %> -
-
-
+
+
+

Worker WebSocket

+ ab-av1 +
+ +
+
+
URL
+
+ {@worker_socket_url}?token=<worker-token>
-
- Syncing {@service_type}... {@sync_progress}% + +
+
Token
+
+ <%= if @worker_token do %> + {@worker_token} + <% else %> + not configured + <% end %> +
- <% end %> +
""" end @@ -1224,6 +1257,8 @@ defmodule ReencodarrWeb.DashboardLive do syncing={@syncing} sync_progress={@sync_progress} service_type={@service_type} + worker_token={@worker_token} + worker_socket_url={@worker_socket_url} />
@@ -1385,6 +1420,13 @@ defmodule ReencodarrWeb.DashboardLive do Process.send_after(self(), :update_dashboard_data, 5_000) end + defp worker_socket_url do + ReencodarrWeb.Endpoint.url() + |> String.replace_prefix("https://", "wss://") + |> String.replace_prefix("http://", "ws://") + |> Kernel.<>("/workers/socket/websocket") + end + defp stats_display(nil) do %{ total_videos: "—", diff --git a/test/reencodarr_web/channels/worker_channel_test.exs b/test/reencodarr_web/channels/worker_channel_test.exs index 54389ae4..50bd3513 100644 --- a/test/reencodarr_web/channels/worker_channel_test.exs +++ b/test/reencodarr_web/channels/worker_channel_test.exs @@ -43,12 +43,7 @@ defmodule ReencodarrWeb.WorkerChannelTest do assert {:ok, socket} = connect(WorkerSocket, %{"token" => token}) assert {:ok, _join_payload, socket} = subscribe_and_join(socket, "workers:crf_search") - assert_reply push(socket, "announce", %{ - "worker_id" => "abav1-dev", - "protocol_version" => 99, - "version" => "0.10.0", - "capabilities" => %{"crf_search" => true} - }), + assert_reply push(socket, "announce", announce_payload(protocol_version: 99)), :error, %{reason: "unsupported_protocol_version", supported_protocol_versions: [1]} after @@ -62,42 +57,58 @@ defmodule ReencodarrWeb.WorkerChannelTest do assert {:ok, socket1} = connect(WorkerSocket, %{"token" => token}) assert {:ok, _join_payload, socket1} = subscribe_and_join(socket1, "workers:crf_search") - assert_reply push(socket1, "announce", %{ - "worker_id" => "abav1-dev", - "protocol_version" => 1, - "version" => "0.10.0", - "capabilities" => %{"crf_search" => true} - }), + assert_reply push(socket1, "announce", announce_payload()), :ok, %{accepted: true, protocol_version: 1} assert {:ok, socket2} = connect(WorkerSocket, %{"token" => token}) assert {:ok, _join_payload, socket2} = subscribe_and_join(socket2, "workers:crf_search") - assert_reply push(socket2, "announce", %{ - "worker_id" => "abav1-dev", - "protocol_version" => 1, - "version" => "0.10.0", - "capabilities" => %{"crf_search" => true} - }), + assert_reply push(socket2, "announce", announce_payload()), :error, %{reason: "duplicate_worker_id"} Process.unlink(socket1.channel_pid) assert :ok = close(socket1) - assert_reply push(socket2, "announce", %{ - "worker_id" => "abav1-dev", - "protocol_version" => 1, - "version" => "0.10.0", - "capabilities" => %{"crf_search" => true} - }), + assert_reply push(socket2, "announce", announce_payload()), :ok, %{accepted: true, protocol_version: 1} after Application.delete_env(:reencodarr, :worker_token) end + test "tracks heartbeat last-seen for announced sessions" do + token = "test-worker-token" + Application.put_env(:reencodarr, :worker_token, token) + + assert {:ok, socket} = connect(WorkerSocket, %{"token" => token}) + assert {:ok, _join_payload, socket} = subscribe_and_join(socket, "workers:crf_search") + + assert_reply push(socket, "announce", announce_payload()), :ok, %{ + accepted: true, + protocol_version: 1 + } + + [session] = WorkerSessions.list() + connected_at = session.connected_at + first_seen_at = session.last_seen_at + + assert_reply push(socket, "heartbeat", %{}), :ok, %{ + accepted: true, + last_seen_at: last_seen_at + } + + assert {:ok, parsed_last_seen_at, 0} = DateTime.from_iso8601(last_seen_at) + + [updated_session] = WorkerSessions.list() + assert updated_session.connected_at == connected_at + assert DateTime.compare(updated_session.last_seen_at, first_seen_at) in [:eq, :gt] + assert updated_session.last_seen_at == parsed_last_seen_at + after + Application.delete_env(:reencodarr, :worker_token) + end + test "rejects missing or invalid tokens" do Application.put_env(:reencodarr, :worker_token, "test-worker-token") @@ -107,4 +118,24 @@ defmodule ReencodarrWeb.WorkerChannelTest do Application.delete_env(:reencodarr, :worker_token) end end + + defp announce_payload(overrides \\ []) do + override_map = + overrides + |> Enum.into(%{}) + |> Map.new(fn + {key, value} when is_atom(key) -> {Atom.to_string(key), value} + pair -> pair + end) + + Map.merge( + %{ + "worker_id" => "abav1-dev", + "protocol_version" => 1, + "version" => "0.10.0", + "capabilities" => %{"crf_search" => true} + }, + override_map + ) + end end diff --git a/test/reencodarr_web/live/dashboard_v2_live_test.exs b/test/reencodarr_web/live/dashboard_v2_live_test.exs index 31daa867..e1068f92 100644 --- a/test/reencodarr_web/live/dashboard_v2_live_test.exs +++ b/test/reencodarr_web/live/dashboard_v2_live_test.exs @@ -82,6 +82,25 @@ defmodule ReencodarrWeb.DashboardLiveTest do assert html =~ "initial-queue-preview.mkv" end + test "shows the worker websocket token when configured", %{conn: conn} do + previous_token = Application.get_env(:reencodarr, :worker_token) + Application.put_env(:reencodarr, :worker_token, "deploy-test-worker-token") + + on_exit(fn -> + if previous_token do + Application.put_env(:reencodarr, :worker_token, previous_token) + else + Application.delete_env(:reencodarr, :worker_token) + end + end) + + {:ok, _view, html} = live(conn, ~p"/") + + assert html =~ "Worker WebSocket" + assert html =~ "deploy-test-worker-token" + assert html =~ "/workers/socket/websocket?token=" + end + test "includes chart data in the initial html response", %{conn: conn} do {:ok, video} = Fixtures.video_fixture(%{ From 8a7cad8ca32ef99c417c0fcb51c7ce488fa16f70 Mon Sep 17 00:00:00 2001 From: Mika Cohen Date: Tue, 7 Jul 2026 09:12:14 -0600 Subject: [PATCH 08/70] Derive worker token from secret key base --- config/runtime.exs | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/config/runtime.exs b/config/runtime.exs index ad9093c4..d1f2ab5c 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -20,7 +20,18 @@ if System.get_env("PHX_SERVER") do config :reencodarr, ReencodarrWeb.Endpoint, server: true end -if worker_token = System.get_env("REENCODARR_WORKER_TOKEN") do +worker_token = + System.get_env("REENCODARR_WORKER_TOKEN") || + case System.get_env("SECRET_KEY_BASE") do + nil -> + nil + + secret_key_base -> + :crypto.hash(:sha256, secret_key_base <> ":ab-av1-worker-token") + |> Base.url_encode64(padding: false) + end + +if worker_token do config :reencodarr, :worker_token, worker_token end From 9ee25b709f70cb9fe663f978a7c6cd9204ae86cc Mon Sep 17 00:00:00 2001 From: Mika Cohen Date: Tue, 7 Jul 2026 10:34:47 -0600 Subject: [PATCH 09/70] Expire stale worker websocket sessions --- lib/reencodarr/ab_av1/worker_sessions.ex | 47 +++++++++++++ lib/reencodarr/diagnostics.ex | 17 ++++- .../ab_av1/worker_sessions_test.exs | 69 +++++++++++++++++++ .../channels/worker_channel_test.exs | 2 +- 4 files changed, 133 insertions(+), 2 deletions(-) create mode 100644 test/reencodarr/ab_av1/worker_sessions_test.exs diff --git a/lib/reencodarr/ab_av1/worker_sessions.ex b/lib/reencodarr/ab_av1/worker_sessions.ex index 1b417bdd..9893738d 100644 --- a/lib/reencodarr/ab_av1/worker_sessions.ex +++ b/lib/reencodarr/ab_av1/worker_sessions.ex @@ -31,6 +31,10 @@ defmodule Reencodarr.AbAv1.WorkerSessions do GenServer.call(__MODULE__, {:unregister, server_worker_id}) end + def expire_stale(timeout_seconds) when is_integer(timeout_seconds) and timeout_seconds >= 0 do + GenServer.call(__MODULE__, {:expire_stale, timeout_seconds}) + end + def list do GenServer.call(__MODULE__, :list) end @@ -41,6 +45,7 @@ defmodule Reencodarr.AbAv1.WorkerSessions do @impl GenServer def init(_opts) do + schedule_expire_stale() {:ok, %{by_server: %{}, by_client: %{}}} end @@ -83,6 +88,11 @@ defmodule Reencodarr.AbAv1.WorkerSessions do {:reply, :ok, drop_session(state, server_worker_id)} end + def handle_call({:expire_stale, timeout_seconds}, _from, state) do + {expired_sessions, next_state} = expire_stale_sessions(state, timeout_seconds) + {:reply, {:ok, expired_sessions}, next_state} + end + def handle_call(:list, _from, state) do sessions = state.by_server @@ -96,6 +106,13 @@ defmodule Reencodarr.AbAv1.WorkerSessions do {:reply, :ok, %{by_server: %{}, by_client: %{}}} end + @impl GenServer + def handle_info(:expire_stale, state) do + {_expired_sessions, next_state} = expire_stale_sessions(state, timeout_seconds()) + schedule_expire_stale() + {:noreply, next_state} + end + defp build_session(attrs, now) do %{ server_worker_id: Map.fetch!(attrs, :server_worker_id), @@ -132,6 +149,36 @@ defmodule Reencodarr.AbAv1.WorkerSessions do end end + defp expire_stale_sessions(state, timeout_seconds) do + now = now() + + Enum.reduce(state.by_server, {[], state}, fn {server_worker_id, session}, + {expired, acc_state} -> + age_seconds = DateTime.diff(now, session.last_seen_at, :second) + + if age_seconds >= timeout_seconds do + {[session | expired], drop_session(acc_state, server_worker_id)} + else + {expired, acc_state} + end + end) + |> then(fn {expired_sessions, next_state} -> + {Enum.reverse(expired_sessions), next_state} + end) + end + + defp schedule_expire_stale do + Process.send_after(self(), :expire_stale, sweep_interval_ms()) + end + + defp timeout_seconds do + Application.get_env(:reencodarr, :worker_session_timeout_seconds, 120) + end + + defp sweep_interval_ms do + Application.get_env(:reencodarr, :worker_session_sweep_interval_ms, 30_000) + end + defp now do DateTime.utc_now() |> DateTime.truncate(:second) end diff --git a/lib/reencodarr/diagnostics.ex b/lib/reencodarr/diagnostics.ex index b5919213..bd90b9be 100644 --- a/lib/reencodarr/diagnostics.ex +++ b/lib/reencodarr/diagnostics.ex @@ -8,7 +8,7 @@ defmodule Reencodarr.Diagnostics do import Ecto.Query - alias Reencodarr.AbAv1.{CrfSearch, Encode} + alias Reencodarr.AbAv1.{CrfSearch, Encode, WorkerSessions} alias Reencodarr.Analyzer alias Reencodarr.Analyzer.MediaInfoCache alias Reencodarr.Core.Time @@ -306,6 +306,7 @@ defmodule Reencodarr.Diagnostics do encode_state = safe_get_state(Reencodarr.AbAv1.Encode, 2000) health_state = safe_get_state(Reencodarr.Encoder.HealthCheck, 2000) cache_stats = safe_call(fn -> MediaInfoCache.get_stats() end) + worker_sessions = safe_call(fn -> WorkerSessions.list() end) """ #{section("Live Process State")} @@ -319,6 +320,9 @@ defmodule Reencodarr.Diagnostics do Health Check: #{format_health_check_state(health_state)} + Worker Sessions: + #{format_worker_sessions(worker_sessions)} + MediaInfo Cache: #{format_cache_stats(cache_stats)} """ @@ -654,6 +658,17 @@ defmodule Reencodarr.Diagnostics do defp format_health_check_state(_), do: " Unknown state" + defp format_worker_sessions({:error, _}), do: " Unavailable" + defp format_worker_sessions([]), do: " none" + + defp format_worker_sessions(worker_sessions) when is_list(worker_sessions) do + Enum.map_join(worker_sessions, "\n", fn session -> + " #{session.client_worker_id} protocol=#{session.protocol_version} version=#{session.version} capabilities=#{inspect(session.capabilities)} last_seen=#{DateTime.to_iso8601(session.last_seen_at)}" + end) + end + + defp format_worker_sessions(_), do: " Unknown state" + defp video_active_in_state?({:error, _}, _video_id), do: false defp video_active_in_state?(state, video_id) when is_map(state) do diff --git a/test/reencodarr/ab_av1/worker_sessions_test.exs b/test/reencodarr/ab_av1/worker_sessions_test.exs new file mode 100644 index 00000000..f95bd38f --- /dev/null +++ b/test/reencodarr/ab_av1/worker_sessions_test.exs @@ -0,0 +1,69 @@ +defmodule Reencodarr.AbAv1.WorkerSessionsTest do + use Reencodarr.DataCase, async: false + + alias Reencodarr.AbAv1.WorkerSessions + alias Reencodarr.Diagnostics + + setup do + WorkerSessions.reset() + :ok + end + + test "expires stale worker sessions" do + assert {:ok, _session} = + WorkerSessions.register(%{ + server_worker_id: "worker-server-1", + client_worker_id: "worker-client-1", + protocol_version: 1, + version: "0.10.0", + capabilities: %{"crf_search" => true} + }) + + assert [_session] = WorkerSessions.list() + + assert {:ok, expired_sessions} = WorkerSessions.expire_stale(0) + assert Enum.map(expired_sessions, & &1.client_worker_id) == ["worker-client-1"] + assert WorkerSessions.list() == [] + end + + test "timer-driven stale expiry removes old sessions" do + previous_timeout = Application.get_env(:reencodarr, :worker_session_timeout_seconds) + Application.put_env(:reencodarr, :worker_session_timeout_seconds, 0) + + on_exit(fn -> + Application.put_env(:reencodarr, :worker_session_timeout_seconds, previous_timeout) + end) + + assert {:ok, _session} = + WorkerSessions.register(%{ + server_worker_id: "worker-server-1", + client_worker_id: "worker-client-1", + protocol_version: 1, + version: "0.10.0", + capabilities: %{"crf_search" => true} + }) + + send(Process.whereis(WorkerSessions), :expire_stale) + :timer.sleep(25) + + assert WorkerSessions.list() == [] + end + + test "includes worker sessions in diagnostics output" do + assert {:ok, _session} = + WorkerSessions.register(%{ + server_worker_id: "worker-server-1", + client_worker_id: "worker-client-1", + protocol_version: 1, + version: "0.10.0", + capabilities: %{"crf_search" => true} + }) + + output = Diagnostics.processes() + + assert output =~ "Worker Sessions:" + assert output =~ "worker-client-1" + assert output =~ "protocol=1" + assert output =~ "version=0.10.0" + end +end diff --git a/test/reencodarr_web/channels/worker_channel_test.exs b/test/reencodarr_web/channels/worker_channel_test.exs index 50bd3513..24e8ba58 100644 --- a/test/reencodarr_web/channels/worker_channel_test.exs +++ b/test/reencodarr_web/channels/worker_channel_test.exs @@ -1,5 +1,5 @@ defmodule ReencodarrWeb.WorkerChannelTest do - use ReencodarrWeb.ChannelCase, async: true + use ReencodarrWeb.ChannelCase, async: false alias Reencodarr.AbAv1.WorkerSessions alias ReencodarrWeb.WorkerSocket From 0f11f0399ff91dbff352c1c971f38690b7e1c326 Mon Sep 17 00:00:00 2001 From: Mika Cohen Date: Tue, 7 Jul 2026 10:49:46 -0600 Subject: [PATCH 10/70] Add pull-based worker assignment --- lib/reencodarr/ab_av1/worker_protocol.ex | 8 ++ lib/reencodarr/ab_av1/worker_sessions.ex | 42 +++++++++ lib/reencodarr/diagnostics.ex | 4 +- lib/reencodarr/media.ex | 4 + lib/reencodarr/media/video_queries.ex | 69 ++++++++++++++ lib/reencodarr_web/channels/worker_channel.ex | 66 +++++++++++++- .../ab_av1/worker_sessions_test.exs | 17 ++++ .../channels/worker_channel_test.exs | 89 +++++++++++++++++++ 8 files changed, 295 insertions(+), 4 deletions(-) diff --git a/lib/reencodarr/ab_av1/worker_protocol.ex b/lib/reencodarr/ab_av1/worker_protocol.ex index 1ba4a233..bd9b70b5 100644 --- a/lib/reencodarr/ab_av1/worker_protocol.ex +++ b/lib/reencodarr/ab_av1/worker_protocol.ex @@ -39,6 +39,14 @@ defmodule Reencodarr.AbAv1.WorkerProtocol do def no_work, do: %{status: "no_work"} + def work_assigned(%{id: video_id, target_vmaf: target_vmaf}) when is_integer(video_id) do + %{status: "assigned", video_id: video_id, target_vmaf: target_vmaf} + end + + def work_assigned(video_id, target_vmaf) when is_integer(video_id) do + %{status: "assigned", video_id: video_id, target_vmaf: target_vmaf} + end + def heartbeat_ack(last_seen_at), do: %{accepted: true, last_seen_at: DateTime.to_iso8601(last_seen_at)} diff --git a/lib/reencodarr/ab_av1/worker_sessions.ex b/lib/reencodarr/ab_av1/worker_sessions.ex index 9893738d..cef52034 100644 --- a/lib/reencodarr/ab_av1/worker_sessions.ex +++ b/lib/reencodarr/ab_av1/worker_sessions.ex @@ -1,3 +1,5 @@ +# credo:disable-for-this-file + defmodule Reencodarr.AbAv1.WorkerSessions do @moduledoc """ Tracks connected ab-av1 worker websocket sessions. @@ -11,6 +13,7 @@ defmodule Reencodarr.AbAv1.WorkerSessions do version: String.t(), protocol_version: pos_integer(), capabilities: map(), + active_video_id: integer() | nil, connected_at: DateTime.t(), last_seen_at: DateTime.t() } @@ -31,6 +34,18 @@ defmodule Reencodarr.AbAv1.WorkerSessions do GenServer.call(__MODULE__, {:unregister, server_worker_id}) end + def assign_video(server_worker_id, video_id) when is_integer(video_id) do + GenServer.call(__MODULE__, {:assign_video, server_worker_id, video_id}) + end + + def clear_video(server_worker_id) do + GenServer.call(__MODULE__, {:clear_video, server_worker_id}) + end + + def get(server_worker_id) do + GenServer.call(__MODULE__, {:get, server_worker_id}) + end + def expire_stale(timeout_seconds) when is_integer(timeout_seconds) and timeout_seconds >= 0 do GenServer.call(__MODULE__, {:expire_stale, timeout_seconds}) end @@ -88,6 +103,32 @@ defmodule Reencodarr.AbAv1.WorkerSessions do {:reply, :ok, drop_session(state, server_worker_id)} end + def handle_call({:assign_video, server_worker_id, video_id}, _from, state) do + case Map.fetch(state.by_server, server_worker_id) do + {:ok, session} -> + updated_session = %{session | active_video_id: video_id} + {:reply, {:ok, updated_session}, put_session(state, updated_session)} + + :error -> + {:reply, {:error, :unknown_worker_session}, state} + end + end + + def handle_call({:clear_video, server_worker_id}, _from, state) do + case Map.fetch(state.by_server, server_worker_id) do + {:ok, session} -> + updated_session = %{session | active_video_id: nil} + {:reply, {:ok, updated_session}, put_session(state, updated_session)} + + :error -> + {:reply, {:error, :unknown_worker_session}, state} + end + end + + def handle_call({:get, server_worker_id}, _from, state) do + {:reply, Map.get(state.by_server, server_worker_id), state} + end + def handle_call({:expire_stale, timeout_seconds}, _from, state) do {expired_sessions, next_state} = expire_stale_sessions(state, timeout_seconds) {:reply, {:ok, expired_sessions}, next_state} @@ -120,6 +161,7 @@ defmodule Reencodarr.AbAv1.WorkerSessions do version: Map.fetch!(attrs, :version), protocol_version: Map.fetch!(attrs, :protocol_version), capabilities: Map.fetch!(attrs, :capabilities), + active_video_id: nil, connected_at: now, last_seen_at: now } diff --git a/lib/reencodarr/diagnostics.ex b/lib/reencodarr/diagnostics.ex index bd90b9be..c9223440 100644 --- a/lib/reencodarr/diagnostics.ex +++ b/lib/reencodarr/diagnostics.ex @@ -663,7 +663,9 @@ defmodule Reencodarr.Diagnostics do defp format_worker_sessions(worker_sessions) when is_list(worker_sessions) do Enum.map_join(worker_sessions, "\n", fn session -> - " #{session.client_worker_id} protocol=#{session.protocol_version} version=#{session.version} capabilities=#{inspect(session.capabilities)} last_seen=#{DateTime.to_iso8601(session.last_seen_at)}" + active_video_id = session.active_video_id || "none" + + " #{session.client_worker_id} protocol=#{session.protocol_version} version=#{session.version} video=#{active_video_id} capabilities=#{inspect(session.capabilities)} last_seen=#{DateTime.to_iso8601(session.last_seen_at)}" end) end diff --git a/lib/reencodarr/media.ex b/lib/reencodarr/media.ex index 0ccdc7ea..bfeb3ab4 100644 --- a/lib/reencodarr/media.ex +++ b/lib/reencodarr/media.ex @@ -158,6 +158,10 @@ defmodule Reencodarr.Media do VideoQueries.videos_for_crf_search(limit) end + def claim_next_video_for_crf_search do + VideoQueries.claim_next_video_for_crf_search() + end + def count_videos_for_crf_search do VideoQueries.count_videos_for_crf_search() end diff --git a/lib/reencodarr/media/video_queries.ex b/lib/reencodarr/media/video_queries.ex index 02efefb0..c6239bd2 100644 --- a/lib/reencodarr/media/video_queries.ex +++ b/lib/reencodarr/media/video_queries.ex @@ -46,6 +46,38 @@ defmodule Reencodarr.Media.VideoQueries do ) end + @doc """ + Atomically claims the next video ready for CRF search. + + Returns `nil` when the queue is empty. The claim transitions the selected + video from `:analyzed` to `:crf_searching` so only one worker can receive it. + """ + @spec claim_next_video_for_crf_search(keyword()) :: Video.t() | nil + def claim_next_video_for_crf_search(opts \\ []) do + DbWriter.transaction( + fn -> + claim_next_video_for_crf_search_in_tx(opts) + end, + label: :video_queries_claim_next_video_for_crf_search + ) + |> case do + {:ok, {:claimed, video, old_snapshot}} -> + Reencodarr.Media.broadcast_video_mutation( + :update, + old_snapshot, + Reencodarr.Media.fetch_dashboard_video_snapshot_by_id(video.id) + ) + + video + + {:ok, :none} -> + nil + + {:error, reason} -> + raise "Failed to claim video for CRF search: #{inspect(reason)}" + end + end + @doc """ Gets videos needing analysis (state: needs_analysis). @@ -156,6 +188,43 @@ defmodule Reencodarr.Media.VideoQueries do end) end + defp claim_next_video_for_crf_search_in_tx(opts) do + candidate_ids = + from(v in Video, + where: v.state == :analyzed, + order_by: [desc: v.priority, desc: v.bitrate, desc: v.size, asc: v.updated_at], + limit: 1, + select: v.id + ) + |> Repo.all(opts) + + claim_next_video_for_crf_search_in_tx(candidate_ids, opts) + end + + defp claim_next_video_for_crf_search_in_tx([], _opts), do: :none + + defp claim_next_video_for_crf_search_in_tx([video_id | rest], opts) do + old_snapshot = Reencodarr.Media.fetch_dashboard_video_snapshot_by_id(video_id) + + {updated_count, _} = + from(v in Video, + where: v.id == ^video_id and v.state == :analyzed, + select: v + ) + |> Repo.update_all([set: [state: :crf_searching, updated_at: DateTime.utc_now()]], opts) + + case updated_count do + 1 -> + case Repo.get(Video, video_id, opts) do + %Video{} = video -> {:claimed, video, old_snapshot} + nil -> :none + end + + _ -> + claim_next_video_for_crf_search_in_tx(rest, opts) + end + end + @doc """ Counts the total number of videos needing analysis. """ diff --git a/lib/reencodarr_web/channels/worker_channel.ex b/lib/reencodarr_web/channels/worker_channel.ex index 6c5c9109..7f902b92 100644 --- a/lib/reencodarr_web/channels/worker_channel.ex +++ b/lib/reencodarr_web/channels/worker_channel.ex @@ -6,6 +6,7 @@ defmodule ReencodarrWeb.WorkerChannel do use ReencodarrWeb, :channel alias Reencodarr.AbAv1.{WorkerProtocol, WorkerSessions} + alias Reencodarr.Media @crf_search_topic WorkerProtocol.crf_search_topic() @@ -61,12 +62,71 @@ defmodule ReencodarrWeb.WorkerChannel do end end - def handle_in("pull_work", _payload, socket), - do: {:reply, {:ok, WorkerProtocol.no_work()}, socket} + def handle_in("request_work", payload, socket), do: handle_work_request(payload, socket) + + def handle_in("pull_work", payload, socket), do: handle_work_request(payload, socket) @impl true - def terminate(_reason, %{assigns: %{worker_id: worker_id}}) do + def terminate(_reason, %{assigns: %{worker_id: worker_id}} = socket) do + maybe_requeue_active_video(socket) WorkerSessions.unregister(worker_id) :ok end + + defp handle_work_request(_payload, %{assigns: %{worker_id: worker_id}} = socket) do + case socket.assigns[:current_video_id] do + nil -> + claim_work(worker_id, socket) + + video_id -> + {:reply, + {:ok, WorkerProtocol.work_assigned(video_id, socket.assigns[:current_vmaf_target])}, + socket} + end + end + + defp claim_work(worker_id, socket) do + case Media.claim_next_video_for_crf_search() do + %Media.Video{} = video -> + assign_claimed_work(worker_id, socket, video) + + nil -> + {:reply, {:ok, WorkerProtocol.no_work()}, socket} + end + end + + defp assign_claimed_work(worker_id, socket, video) do + target_vmaf = Reencodarr.Rules.vmaf_target(video) + + case WorkerSessions.assign_video(worker_id, video.id) do + {:ok, _session} -> + socket = + socket + |> assign(:current_video_id, video.id) + |> assign(:current_vmaf_target, target_vmaf) + + {:reply, {:ok, WorkerProtocol.work_assigned(video.id, target_vmaf)}, socket} + + {:error, reason} -> + _ = Media.mark_as_analyzed(video) + {:reply, {:error, WorkerProtocol.error(reason)}, socket} + end + end + + defp maybe_requeue_active_video(socket) do + case socket.assigns[:current_video_id] do + nil -> + :ok + + video_id -> + case Media.get_video(video_id) do + %Media.Video{state: :crf_searching} = video -> + _ = Media.mark_as_analyzed(video) + :ok + + _ -> + :ok + end + end + end end diff --git a/test/reencodarr/ab_av1/worker_sessions_test.exs b/test/reencodarr/ab_av1/worker_sessions_test.exs index f95bd38f..b3fbd32f 100644 --- a/test/reencodarr/ab_av1/worker_sessions_test.exs +++ b/test/reencodarr/ab_av1/worker_sessions_test.exs @@ -66,4 +66,21 @@ defmodule Reencodarr.AbAv1.WorkerSessionsTest do assert output =~ "protocol=1" assert output =~ "version=0.10.0" end + + test "tracks an assigned video on the session" do + assert {:ok, _session} = + WorkerSessions.register(%{ + server_worker_id: "worker-server-1", + client_worker_id: "worker-client-1", + protocol_version: 1, + version: "0.10.0", + capabilities: %{"crf_search" => true} + }) + + assert {:ok, session} = WorkerSessions.assign_video("worker-server-1", 123) + assert session.active_video_id == 123 + + [listed_session] = WorkerSessions.list() + assert listed_session.active_video_id == 123 + end end diff --git a/test/reencodarr_web/channels/worker_channel_test.exs b/test/reencodarr_web/channels/worker_channel_test.exs index 24e8ba58..01d02310 100644 --- a/test/reencodarr_web/channels/worker_channel_test.exs +++ b/test/reencodarr_web/channels/worker_channel_test.exs @@ -2,6 +2,8 @@ defmodule ReencodarrWeb.WorkerChannelTest do use ReencodarrWeb.ChannelCase, async: false alias Reencodarr.AbAv1.WorkerSessions + alias Reencodarr.Fixtures + alias Reencodarr.Media alias ReencodarrWeb.WorkerSocket describe "ab-av1 worker websocket" do @@ -36,6 +38,31 @@ defmodule ReencodarrWeb.WorkerChannelTest do Application.delete_env(:reencodarr, :worker_token) end + test "assigns one queued video to a worker and marks it crf_searching" do + token = "test-worker-token" + Application.put_env(:reencodarr, :worker_token, token) + + {:ok, video} = Fixtures.video_fixture(%{state: :analyzed}) + + assert {:ok, socket} = connect(WorkerSocket, %{"token" => token}) + assert {:ok, _join_payload, socket} = subscribe_and_join(socket, "workers:crf_search") + + assert_reply push(socket, "announce", announce_payload(worker_id: "worker-a")), + :ok, + %{accepted: true, protocol_version: 1} + + assert_reply push(socket, "pull_work", %{}), + :ok, + %{status: "assigned", video_id: assigned_video_id} + + assert assigned_video_id == video.id + assert Media.get_video(video.id).state == :crf_searching + assert [session] = WorkerSessions.list() + assert session.active_video_id == video.id + after + Application.delete_env(:reencodarr, :worker_token) + end + test "rejects unsupported protocol versions" do token = "test-worker-token" Application.put_env(:reencodarr, :worker_token, token) @@ -78,6 +105,39 @@ defmodule ReencodarrWeb.WorkerChannelTest do Application.delete_env(:reencodarr, :worker_token) end + test "does not hand the same queued video to two workers" do + token = "test-worker-token" + Application.put_env(:reencodarr, :worker_token, token) + + {:ok, video} = Fixtures.video_fixture(%{state: :analyzed}) + + assert {:ok, socket1} = connect(WorkerSocket, %{"token" => token}) + assert {:ok, _join_payload, socket1} = subscribe_and_join(socket1, "workers:crf_search") + + assert_reply push(socket1, "announce", announce_payload(worker_id: "worker-a")), + :ok, + %{accepted: true, protocol_version: 1} + + assert {:ok, socket2} = connect(WorkerSocket, %{"token" => token}) + assert {:ok, _join_payload, socket2} = subscribe_and_join(socket2, "workers:crf_search") + + assert_reply push(socket2, "announce", announce_payload(worker_id: "worker-b")), + :ok, + %{accepted: true, protocol_version: 1} + + assert_reply push(socket1, "pull_work", %{}), + :ok, + %{status: "assigned", video_id: assigned_video_id} + + assert assigned_video_id == video.id + + assert_reply push(socket2, "pull_work", %{}), + :ok, + %{status: "no_work"} + after + Application.delete_env(:reencodarr, :worker_token) + end + test "tracks heartbeat last-seen for announced sessions" do token = "test-worker-token" Application.put_env(:reencodarr, :worker_token, token) @@ -109,6 +169,35 @@ defmodule ReencodarrWeb.WorkerChannelTest do Application.delete_env(:reencodarr, :worker_token) end + test "requeues active work when the worker disconnects" do + token = "test-worker-token" + Application.put_env(:reencodarr, :worker_token, token) + + {:ok, video} = Fixtures.video_fixture(%{state: :analyzed}) + + assert {:ok, socket} = connect(WorkerSocket, %{"token" => token}) + assert {:ok, _join_payload, socket} = subscribe_and_join(socket, "workers:crf_search") + + assert_reply push(socket, "announce", announce_payload(worker_id: "worker-a")), + :ok, + %{accepted: true, protocol_version: 1} + + assert_reply push(socket, "pull_work", %{}), + :ok, + %{status: "assigned", video_id: assigned_video_id} + + assert assigned_video_id == video.id + assert Media.get_video(video.id).state == :crf_searching + + Process.unlink(socket.channel_pid) + assert :ok = close(socket) + + assert Media.get_video(video.id).state == :analyzed + assert WorkerSessions.list() == [] + after + Application.delete_env(:reencodarr, :worker_token) + end + test "rejects missing or invalid tokens" do Application.put_env(:reencodarr, :worker_token, "test-worker-token") From 803f59f9a98bee77583528bb31922336da2a2ec1 Mon Sep 17 00:00:00 2001 From: Mika Cohen Date: Tue, 7 Jul 2026 10:58:03 -0600 Subject: [PATCH 11/70] Use ETS for worker sessions --- docs/ab_av1_worker_websocket.md | 16 ++ lib/reencodarr/ab_av1/worker_protocol.ex | 17 +- lib/reencodarr/ab_av1/worker_sessions.ex | 150 +++++++++++------- lib/reencodarr_web/channels/worker_channel.ex | 14 +- .../ab_av1/worker_protocol_test.exs | 27 ++++ .../channels/worker_channel_test.exs | 18 ++- 6 files changed, 172 insertions(+), 70 deletions(-) create mode 100644 test/reencodarr/ab_av1/worker_protocol_test.exs diff --git a/docs/ab_av1_worker_websocket.md b/docs/ab_av1_worker_websocket.md index 0e61f50d..d0b24434 100644 --- a/docs/ab_av1_worker_websocket.md +++ b/docs/ab_av1_worker_websocket.md @@ -42,4 +42,20 @@ Current first-pass reply when no distributed job is available: {"status": "no_work"} ``` +Current assigned-work reply: + +```json +{ + "status": "job_assigned", + "job_id": "123", + "video_id": 123, + "source_name": "movie.mkv", + "size_bytes": 987654, + "chunk_size_bytes": 1048576, + "target_vmaf": 96.5 +} +``` + +Workers only run one CRF search or encode at a time. + File transfer, chunk checksums, progress, result reporting, and requeue/failure handling are still pending REENC-133 follow-up slices. diff --git a/lib/reencodarr/ab_av1/worker_protocol.ex b/lib/reencodarr/ab_av1/worker_protocol.ex index bd9b70b5..fbc25eef 100644 --- a/lib/reencodarr/ab_av1/worker_protocol.ex +++ b/lib/reencodarr/ab_av1/worker_protocol.ex @@ -39,12 +39,17 @@ defmodule Reencodarr.AbAv1.WorkerProtocol do def no_work, do: %{status: "no_work"} - def work_assigned(%{id: video_id, target_vmaf: target_vmaf}) when is_integer(video_id) do - %{status: "assigned", video_id: video_id, target_vmaf: target_vmaf} - end - - def work_assigned(video_id, target_vmaf) when is_integer(video_id) do - %{status: "assigned", video_id: video_id, target_vmaf: target_vmaf} + def work_assigned(%{id: video_id, path: path, size: size}, target_vmaf) + when is_integer(video_id) and is_binary(path) do + %{ + status: "job_assigned", + job_id: Integer.to_string(video_id), + video_id: video_id, + source_name: Path.basename(path), + size_bytes: size || 0, + chunk_size_bytes: 1_048_576, + target_vmaf: target_vmaf + } end def heartbeat_ack(last_seen_at), diff --git a/lib/reencodarr/ab_av1/worker_sessions.ex b/lib/reencodarr/ab_av1/worker_sessions.ex index cef52034..a39301cd 100644 --- a/lib/reencodarr/ab_av1/worker_sessions.ex +++ b/lib/reencodarr/ab_av1/worker_sessions.ex @@ -1,5 +1,3 @@ -# credo:disable-for-this-file - defmodule Reencodarr.AbAv1.WorkerSessions do @moduledoc """ Tracks connected ab-av1 worker websocket sessions. @@ -7,6 +5,9 @@ defmodule Reencodarr.AbAv1.WorkerSessions do use GenServer + @by_server_table :reencodarr_worker_sessions_by_server + @by_client_table :reencodarr_worker_sessions_by_client + @type session :: %{ server_worker_id: String.t(), client_worker_id: String.t(), @@ -60,8 +61,10 @@ defmodule Reencodarr.AbAv1.WorkerSessions do @impl GenServer def init(_opts) do + :ets.new(@by_server_table, [:named_table, :set, :private]) + :ets.new(@by_client_table, [:named_table, :set, :private]) schedule_expire_stale() - {:ok, %{by_server: %{}, by_client: %{}}} + {:ok, :ok} end @impl GenServer @@ -70,18 +73,22 @@ defmodule Reencodarr.AbAv1.WorkerSessions do server_worker_id = Map.fetch!(attrs, :server_worker_id) client_worker_id = Map.fetch!(attrs, :client_worker_id) - case Map.get(state.by_client, client_worker_id) do + case lookup_client(client_worker_id) do nil -> session = build_session(attrs, now) - {:reply, {:ok, session}, put_session(state, session)} + :ok = put_session(session) + {:reply, {:ok, session}, state} ^server_worker_id -> + connected_at = lookup_session!(server_worker_id).connected_at + session = attrs |> build_session(now) - |> Map.put(:connected_at, state.by_server[server_worker_id].connected_at) + |> Map.put(:connected_at, connected_at) - {:reply, {:ok, session}, put_session(state, session)} + :ok = put_session(session) + {:reply, {:ok, session}, state} _other_server_worker_id -> {:reply, {:error, :duplicate_worker_id}, state} @@ -89,10 +96,11 @@ defmodule Reencodarr.AbAv1.WorkerSessions do end def handle_call({:touch, server_worker_id}, _from, state) do - case Map.fetch(state.by_server, server_worker_id) do + case lookup_session(server_worker_id) do {:ok, session} -> updated_session = %{session | last_seen_at: now()} - {:reply, {:ok, updated_session}, put_session(state, updated_session)} + :ok = put_session(updated_session) + {:reply, {:ok, updated_session}, state} :error -> {:reply, {:error, :unknown_worker_session}, state} @@ -100,14 +108,16 @@ defmodule Reencodarr.AbAv1.WorkerSessions do end def handle_call({:unregister, server_worker_id}, _from, state) do - {:reply, :ok, drop_session(state, server_worker_id)} + :ok = drop_session(server_worker_id) + {:reply, :ok, state} end def handle_call({:assign_video, server_worker_id, video_id}, _from, state) do - case Map.fetch(state.by_server, server_worker_id) do + case lookup_session(server_worker_id) do {:ok, session} -> updated_session = %{session | active_video_id: video_id} - {:reply, {:ok, updated_session}, put_session(state, updated_session)} + :ok = put_session(updated_session) + {:reply, {:ok, updated_session}, state} :error -> {:reply, {:error, :unknown_worker_session}, state} @@ -115,10 +125,11 @@ defmodule Reencodarr.AbAv1.WorkerSessions do end def handle_call({:clear_video, server_worker_id}, _from, state) do - case Map.fetch(state.by_server, server_worker_id) do + case lookup_session(server_worker_id) do {:ok, session} -> updated_session = %{session | active_video_id: nil} - {:reply, {:ok, updated_session}, put_session(state, updated_session)} + :ok = put_session(updated_session) + {:reply, {:ok, updated_session}, state} :error -> {:reply, {:error, :unknown_worker_session}, state} @@ -126,32 +137,29 @@ defmodule Reencodarr.AbAv1.WorkerSessions do end def handle_call({:get, server_worker_id}, _from, state) do - {:reply, Map.get(state.by_server, server_worker_id), state} + {:reply, session_or_nil(lookup_session(server_worker_id)), state} end def handle_call({:expire_stale, timeout_seconds}, _from, state) do - {expired_sessions, next_state} = expire_stale_sessions(state, timeout_seconds) - {:reply, {:ok, expired_sessions}, next_state} + {expired_sessions, _} = expire_stale_sessions(timeout_seconds) + {:reply, {:ok, expired_sessions}, state} end def handle_call(:list, _from, state) do - sessions = - state.by_server - |> Map.values() - |> Enum.sort_by(& &1.client_worker_id) - + sessions = list_sessions() {:reply, sessions, state} end - def handle_call(:reset, _from, _state) do - {:reply, :ok, %{by_server: %{}, by_client: %{}}} + def handle_call(:reset, _from, state) do + :ok = reset_tables() + {:reply, :ok, state} end @impl GenServer def handle_info(:expire_stale, state) do - {_expired_sessions, next_state} = expire_stale_sessions(state, timeout_seconds()) + _ = expire_stale_sessions(timeout_seconds()) schedule_expire_stale() - {:noreply, next_state} + {:noreply, state} end defp build_session(attrs, now) do @@ -167,52 +175,82 @@ defmodule Reencodarr.AbAv1.WorkerSessions do } end - defp put_session(state, session) do - state - |> drop_session(session.server_worker_id) - |> then(fn state -> - %{ - by_server: Map.put(state.by_server, session.server_worker_id, session), - by_client: Map.put(state.by_client, session.client_worker_id, session.server_worker_id) - } - end) + defp put_session(session) do + :ok = drop_session(session.server_worker_id) + true = :ets.insert(@by_server_table, {session.server_worker_id, session}) + true = :ets.insert(@by_client_table, {session.client_worker_id, session.server_worker_id}) + :ok end - defp drop_session(state, server_worker_id) do - case Map.pop(state.by_server, server_worker_id) do - {nil, by_server} -> - %{state | by_server: by_server} + defp drop_session(server_worker_id) do + case lookup_session(server_worker_id) do + {:ok, session} -> + true = :ets.delete(@by_server_table, server_worker_id) + true = :ets.delete(@by_client_table, session.client_worker_id) + :ok - {session, by_server} -> - %{ - by_server: by_server, - by_client: Map.delete(state.by_client, session.client_worker_id) - } + :error -> + :ok end end - defp expire_stale_sessions(state, timeout_seconds) do + defp expire_stale_sessions(timeout_seconds) do now = now() - Enum.reduce(state.by_server, {[], state}, fn {server_worker_id, session}, - {expired, acc_state} -> - age_seconds = DateTime.diff(now, session.last_seen_at, :second) + expired_sessions = + @by_server_table + |> :ets.tab2list() + |> Enum.map(fn {_server_worker_id, session} -> session end) + |> Enum.filter(fn session -> + DateTime.diff(now, session.last_seen_at, :second) >= timeout_seconds + end) - if age_seconds >= timeout_seconds do - {[session | expired], drop_session(acc_state, server_worker_id)} - else - {expired, acc_state} - end - end) - |> then(fn {expired_sessions, next_state} -> - {Enum.reverse(expired_sessions), next_state} + Enum.each(expired_sessions, fn session -> + :ok = drop_session(session.server_worker_id) end) + + {expired_sessions, :ok} end defp schedule_expire_stale do Process.send_after(self(), :expire_stale, sweep_interval_ms()) end + defp reset_tables do + true = :ets.delete_all_objects(@by_server_table) + true = :ets.delete_all_objects(@by_client_table) + :ok + end + + defp list_sessions do + @by_server_table + |> :ets.tab2list() + |> Enum.map(fn {_server_worker_id, session} -> session end) + |> Enum.sort_by(& &1.client_worker_id) + end + + defp lookup_session(server_worker_id) do + case :ets.lookup(@by_server_table, server_worker_id) do + [{^server_worker_id, session}] -> {:ok, session} + [] -> :error + end + end + + defp lookup_session!(server_worker_id) do + {:ok, session} = lookup_session(server_worker_id) + session + end + + defp lookup_client(client_worker_id) do + case :ets.lookup(@by_client_table, client_worker_id) do + [{^client_worker_id, server_worker_id}] -> server_worker_id + [] -> nil + end + end + + defp session_or_nil({:ok, session}), do: session + defp session_or_nil(:error), do: nil + defp timeout_seconds do Application.get_env(:reencodarr, :worker_session_timeout_seconds, 120) end diff --git a/lib/reencodarr_web/channels/worker_channel.ex b/lib/reencodarr_web/channels/worker_channel.ex index 7f902b92..dc0ce490 100644 --- a/lib/reencodarr_web/channels/worker_channel.ex +++ b/lib/reencodarr_web/channels/worker_channel.ex @@ -79,9 +79,15 @@ defmodule ReencodarrWeb.WorkerChannel do claim_work(worker_id, socket) video_id -> - {:reply, - {:ok, WorkerProtocol.work_assigned(video_id, socket.assigns[:current_vmaf_target])}, - socket} + case Media.get_video(video_id) do + %Media.Video{} = video -> + {:reply, + {:ok, WorkerProtocol.work_assigned(video, socket.assigns[:current_vmaf_target])}, + socket} + + nil -> + {:reply, {:error, WorkerProtocol.error(:unknown_worker_session)}, socket} + end end end @@ -105,7 +111,7 @@ defmodule ReencodarrWeb.WorkerChannel do |> assign(:current_video_id, video.id) |> assign(:current_vmaf_target, target_vmaf) - {:reply, {:ok, WorkerProtocol.work_assigned(video.id, target_vmaf)}, socket} + {:reply, {:ok, WorkerProtocol.work_assigned(video, target_vmaf)}, socket} {:error, reason} -> _ = Media.mark_as_analyzed(video) diff --git a/test/reencodarr/ab_av1/worker_protocol_test.exs b/test/reencodarr/ab_av1/worker_protocol_test.exs new file mode 100644 index 00000000..39e61b2d --- /dev/null +++ b/test/reencodarr/ab_av1/worker_protocol_test.exs @@ -0,0 +1,27 @@ +defmodule Reencodarr.AbAv1.WorkerProtocolTest do + use ExUnit.Case, async: true + + alias Reencodarr.AbAv1.WorkerProtocol + + test "builds a job_assigned payload from the claimed video" do + payload = + WorkerProtocol.work_assigned( + %{ + id: 123, + path: "/videos/movie.mkv", + size: 987_654 + }, + 96.5 + ) + + assert payload == %{ + status: "job_assigned", + job_id: "123", + video_id: 123, + source_name: "movie.mkv", + size_bytes: 987_654, + chunk_size_bytes: 1_048_576, + target_vmaf: 96.5 + } + end +end diff --git a/test/reencodarr_web/channels/worker_channel_test.exs b/test/reencodarr_web/channels/worker_channel_test.exs index 01d02310..383698fd 100644 --- a/test/reencodarr_web/channels/worker_channel_test.exs +++ b/test/reencodarr_web/channels/worker_channel_test.exs @@ -53,9 +53,19 @@ defmodule ReencodarrWeb.WorkerChannelTest do assert_reply push(socket, "pull_work", %{}), :ok, - %{status: "assigned", video_id: assigned_video_id} - + %{ + status: "job_assigned", + job_id: job_id, + video_id: assigned_video_id, + source_name: source_name, + size_bytes: size_bytes, + chunk_size_bytes: 1_048_576 + } + + assert job_id == Integer.to_string(video.id) assert assigned_video_id == video.id + assert source_name == Path.basename(video.path) + assert size_bytes == video.size assert Media.get_video(video.id).state == :crf_searching assert [session] = WorkerSessions.list() assert session.active_video_id == video.id @@ -127,7 +137,7 @@ defmodule ReencodarrWeb.WorkerChannelTest do assert_reply push(socket1, "pull_work", %{}), :ok, - %{status: "assigned", video_id: assigned_video_id} + %{status: "job_assigned", video_id: assigned_video_id} assert assigned_video_id == video.id @@ -184,7 +194,7 @@ defmodule ReencodarrWeb.WorkerChannelTest do assert_reply push(socket, "pull_work", %{}), :ok, - %{status: "assigned", video_id: assigned_video_id} + %{status: "job_assigned", video_id: assigned_video_id} assert assigned_video_id == video.id assert Media.get_video(video.id).state == :crf_searching From 0f678614469759ac01e6841f7e04963fcc44a915 Mon Sep 17 00:00:00 2001 From: Mika Cohen Date: Tue, 7 Jul 2026 11:03:42 -0600 Subject: [PATCH 12/70] Hide worker token on dashboard --- docs/ab_av1_worker_websocket.md | 3 +++ lib/reencodarr_web/live/dashboard_live.ex | 13 +++++++++++-- test/reencodarr_web/live/dashboard_v2_live_test.exs | 10 +++++++++- 3 files changed, 23 insertions(+), 3 deletions(-) diff --git a/docs/ab_av1_worker_websocket.md b/docs/ab_av1_worker_websocket.md index d0b24434..19d6b408 100644 --- a/docs/ab_av1_worker_websocket.md +++ b/docs/ab_av1_worker_websocket.md @@ -18,6 +18,7 @@ After joining `workers:crf_search`, send: "event": "announce", "payload": { "worker_id": "abav1-dev", + "protocol_version": 1, "version": "0.10.0", "capabilities": {"crf_search": true} } @@ -58,4 +59,6 @@ Current assigned-work reply: Workers only run one CRF search or encode at a time. +The dashboard shows a token fingerprint instead of the raw worker token. + File transfer, chunk checksums, progress, result reporting, and requeue/failure handling are still pending REENC-133 follow-up slices. diff --git a/lib/reencodarr_web/live/dashboard_live.ex b/lib/reencodarr_web/live/dashboard_live.ex index c7f92578..c82c029a 100644 --- a/lib/reencodarr_web/live/dashboard_live.ex +++ b/lib/reencodarr_web/live/dashboard_live.ex @@ -1051,10 +1051,10 @@ defmodule ReencodarrWeb.DashboardLive do
-
Token
+
Token fingerprint
<%= if @worker_token do %> - {@worker_token} + {worker_token_fingerprint(@worker_token)} <% else %> not configured <% end %> @@ -1081,6 +1081,15 @@ defmodule ReencodarrWeb.DashboardLive do defp format_number(_), do: "—" + defp worker_token_fingerprint(token) when is_binary(token) do + digest = + :crypto.hash(:sha256, token) + |> Base.encode16(case: :lower) + |> String.slice(0, 12) + + "sha256:#{digest}" + end + defp format_completed(stats) do total = stats.total_videos || 0 encoded = stats.encoded || 0 diff --git a/test/reencodarr_web/live/dashboard_v2_live_test.exs b/test/reencodarr_web/live/dashboard_v2_live_test.exs index e1068f92..1e4eb0ae 100644 --- a/test/reencodarr_web/live/dashboard_v2_live_test.exs +++ b/test/reencodarr_web/live/dashboard_v2_live_test.exs @@ -95,9 +95,10 @@ defmodule ReencodarrWeb.DashboardLiveTest do end) {:ok, _view, html} = live(conn, ~p"/") + expected_fingerprint = worker_token_fingerprint("deploy-test-worker-token") assert html =~ "Worker WebSocket" - assert html =~ "deploy-test-worker-token" + assert html =~ expected_fingerprint assert html =~ "/workers/socket/websocket?token=" end @@ -513,4 +514,11 @@ defmodule ReencodarrWeb.DashboardLiveTest do assert html =~ "Unknown sync service" end end + + defp worker_token_fingerprint(token) do + :crypto.hash(:sha256, token) + |> Base.encode16(case: :lower) + |> String.slice(0, 12) + |> then(&"sha256:#{&1}") + end end From 31219d1aaca3416bff0911a803628e98f613a17b Mon Sep 17 00:00:00 2001 From: Mika Cohen Date: Tue, 7 Jul 2026 11:14:41 -0600 Subject: [PATCH 13/70] Claim CRF work before Broadway dispatch --- .../crf_searcher/broadway/producer.ex | 5 ++- test/reencodarr/broadway_producers_test.exs | 41 +++++++++++++++++++ 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/lib/reencodarr/crf_searcher/broadway/producer.ex b/lib/reencodarr/crf_searcher/broadway/producer.ex index a0fdfe9d..e21a21e0 100644 --- a/lib/reencodarr/crf_searcher/broadway/producer.ex +++ b/lib/reencodarr/crf_searcher/broadway/producer.ex @@ -64,7 +64,10 @@ defmodule Reencodarr.CrfSearcher.Broadway.Producer do videos = if status == :available and not suspended? do - Media.get_videos_for_crf_search(1) + case Media.claim_next_video_for_crf_search() do + nil -> [] + video -> [video] + end else [] end diff --git a/test/reencodarr/broadway_producers_test.exs b/test/reencodarr/broadway_producers_test.exs index 895b1ad6..aebb723b 100644 --- a/test/reencodarr/broadway_producers_test.exs +++ b/test/reencodarr/broadway_producers_test.exs @@ -116,6 +116,34 @@ defmodule Reencodarr.BroadwayProducersTest do assert is_list(videos) end + test "atomic claim transitions video to :crf_searching" do + expect_crf_available() + + {:ok, video} = video_fixture(%{state: :analyzed}) + + state = init_state(CrfProducer) + {:noreply, videos, _state2} = CrfProducer.handle_demand(1, state) + + assert Enum.count(videos) == 1 + assert hd(videos).id == video.id + + refreshed = Reencodarr.Repo.get!(Reencodarr.Media.Video, video.id) + assert refreshed.state == :crf_searching + end + + test "claimed CRF videos are not dispatched again" do + expect_crf_available() + + {:ok, _video} = video_fixture(%{state: :analyzed}) + + state = init_state(CrfProducer) + {:noreply, videos1, state2} = CrfProducer.handle_demand(1, state) + assert Enum.count(videos1) == 1 + + {:noreply, videos2, _state3} = CrfProducer.handle_demand(1, state2) + assert videos2 == [] + end + test "poll returns list when called" do {:ok, _video} = video_fixture(%{state: :analyzed}) @@ -168,4 +196,17 @@ defmodule Reencodarr.BroadwayProducersTest do assert {:noreply, [], ^state} = EncoderProducer.handle_info(:unknown, state) end end + + defp expect_crf_available do + :meck.new(Reencodarr.AbAv1.CrfSearch, [:passthrough]) + :meck.expect(Reencodarr.AbAv1.CrfSearch, :available?, fn -> :available end) + + on_exit(fn -> + try do + :meck.unload(Reencodarr.AbAv1.CrfSearch) + catch + :error, {:not_mocked, Reencodarr.AbAv1.CrfSearch} -> :ok + end + end) + end end From b1f402f62d611fcccd4a342cb47b149e2830b61c Mon Sep 17 00:00:00 2001 From: Mika Cohen Date: Tue, 7 Jul 2026 11:43:08 -0600 Subject: [PATCH 14/70] Address ab-av1 websocket review feedback --- config/runtime.exs | 10 +-- docs/ab_av1_worker_websocket.md | 64 ------------------- lib/reencodarr/ab_av1/worker_protocol.ex | 36 ++++++++++- lib/reencodarr/diagnostics.ex | 12 ++-- lib/reencodarr/media/video_queries.ex | 2 +- lib/reencodarr_web/channels/worker_channel.ex | 18 ++++-- nix/dev-shell.nix | 5 ++ nix/module.nix | 9 +++ .../ab_av1/worker_protocol_test.exs | 19 +++++- .../channels/worker_channel_test.exs | 8 +++ 10 files changed, 96 insertions(+), 87 deletions(-) delete mode 100644 docs/ab_av1_worker_websocket.md diff --git a/config/runtime.exs b/config/runtime.exs index d1f2ab5c..9734a4a0 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -21,15 +21,7 @@ if System.get_env("PHX_SERVER") do end worker_token = - System.get_env("REENCODARR_WORKER_TOKEN") || - case System.get_env("SECRET_KEY_BASE") do - nil -> - nil - - secret_key_base -> - :crypto.hash(:sha256, secret_key_base <> ":ab-av1-worker-token") - |> Base.url_encode64(padding: false) - end + System.get_env("REENCODARR_WORKER_TOKEN") if worker_token do config :reencodarr, :worker_token, worker_token diff --git a/docs/ab_av1_worker_websocket.md b/docs/ab_av1_worker_websocket.md deleted file mode 100644 index 19d6b408..00000000 --- a/docs/ab_av1_worker_websocket.md +++ /dev/null @@ -1,64 +0,0 @@ -# ab-av1 Worker Websocket - -First REENC-133 server contract for CRF-search workers. - -## Connection - -- Websocket URL: `/workers/socket/websocket?token=` -- This is served by the main Phoenix app endpoint on the same host/port as the dashboard and API. It is not a separate service port. -- Runtime token env: `REENCODARR_WORKER_TOKEN` -- Phoenix channel topic: `workers:crf_search` - -## Events - -After joining `workers:crf_search`, send: - -```json -{ - "event": "announce", - "payload": { - "worker_id": "abav1-dev", - "protocol_version": 1, - "version": "0.10.0", - "capabilities": {"crf_search": true} - } -} -``` - -Expected reply payload: - -```json -{"accepted": true} -``` - -Then workers can ask for work: - -```json -{"event": "pull_work", "payload": {}} -``` - -Current first-pass reply when no distributed job is available: - -```json -{"status": "no_work"} -``` - -Current assigned-work reply: - -```json -{ - "status": "job_assigned", - "job_id": "123", - "video_id": 123, - "source_name": "movie.mkv", - "size_bytes": 987654, - "chunk_size_bytes": 1048576, - "target_vmaf": 96.5 -} -``` - -Workers only run one CRF search or encode at a time. - -The dashboard shows a token fingerprint instead of the raw worker token. - -File transfer, chunk checksums, progress, result reporting, and requeue/failure handling are still pending REENC-133 follow-up slices. diff --git a/lib/reencodarr/ab_av1/worker_protocol.ex b/lib/reencodarr/ab_av1/worker_protocol.ex index fbc25eef..14a75941 100644 --- a/lib/reencodarr/ab_av1/worker_protocol.ex +++ b/lib/reencodarr/ab_av1/worker_protocol.ex @@ -3,19 +3,40 @@ defmodule Reencodarr.AbAv1.WorkerProtocol do Server-side helpers for the ab-av1 worker websocket protocol. """ + alias Reencodarr.Media.Video + @crf_search_topic "workers:crf_search" @supported_protocol_versions [1] + defmodule Announcement do + @moduledoc false + + @enforce_keys [:worker_id, :protocol_version, :version, :capabilities] + defstruct [:worker_id, :protocol_version, :version, :capabilities] + + @type t :: %__MODULE__{ + worker_id: String.t(), + protocol_version: pos_integer(), + version: String.t(), + capabilities: map() + } + end + + @spec crf_search_topic() :: String.t() def crf_search_topic, do: @crf_search_topic + @spec supported_protocol_versions() :: [pos_integer()] def supported_protocol_versions, do: @supported_protocol_versions + @spec valid_topic?(String.t()) :: boolean() def valid_topic?(@crf_search_topic), do: true def valid_topic?(_topic), do: false + @spec supported_protocol_version?(integer()) :: boolean() def supported_protocol_version?(protocol_version), do: protocol_version in @supported_protocol_versions + @spec parse_announcement(map()) :: {:ok, Announcement.t()} | {:error, :invalid_announcement} def parse_announcement(%{ "worker_id" => worker_id, "protocol_version" => protocol_version, @@ -25,7 +46,7 @@ defmodule Reencodarr.AbAv1.WorkerProtocol do when is_binary(worker_id) and is_integer(protocol_version) and is_binary(version) and is_map(capabilities) do {:ok, - %{ + %Announcement{ worker_id: worker_id, protocol_version: protocol_version, version: version, @@ -35,11 +56,14 @@ defmodule Reencodarr.AbAv1.WorkerProtocol do def parse_announcement(_payload), do: {:error, :invalid_announcement} + @spec accepted(pos_integer()) :: map() def accepted(protocol_version), do: %{accepted: true, protocol_version: protocol_version} + @spec no_work() :: map() def no_work, do: %{status: "no_work"} - def work_assigned(%{id: video_id, path: path, size: size}, target_vmaf) + @spec work_assigned(Video.t(), number()) :: map() + def work_assigned(%Video{id: video_id, path: path, size: size}, target_vmaf) when is_integer(video_id) and is_binary(path) do %{ status: "job_assigned", @@ -52,9 +76,17 @@ defmodule Reencodarr.AbAv1.WorkerProtocol do } end + @spec heartbeat_ack(DateTime.t()) :: map() def heartbeat_ack(last_seen_at), do: %{accepted: true, last_seen_at: DateTime.to_iso8601(last_seen_at)} + @spec error( + :duplicate_worker_id + | :invalid_announcement + | :unsupported_protocol_version + | :unknown_worker_session + | :unauthorized + ) :: map() def error(:duplicate_worker_id), do: %{reason: "duplicate_worker_id"} def error(:invalid_announcement), do: %{reason: "invalid_announcement"} diff --git a/lib/reencodarr/diagnostics.ex b/lib/reencodarr/diagnostics.ex index c9223440..fd3c1a36 100644 --- a/lib/reencodarr/diagnostics.ex +++ b/lib/reencodarr/diagnostics.ex @@ -308,6 +308,13 @@ defmodule Reencodarr.Diagnostics do cache_stats = safe_call(fn -> MediaInfoCache.get_stats() end) worker_sessions = safe_call(fn -> WorkerSessions.list() end) + worker_sessions_section = + case worker_sessions do + {:error, _} -> " Unavailable" + sessions when is_list(sessions) -> format_worker_sessions(sessions) + _ -> " Unknown state" + end + """ #{section("Live Process State")} @@ -321,7 +328,7 @@ defmodule Reencodarr.Diagnostics do #{format_health_check_state(health_state)} Worker Sessions: - #{format_worker_sessions(worker_sessions)} + #{worker_sessions_section} MediaInfo Cache: #{format_cache_stats(cache_stats)} @@ -658,7 +665,6 @@ defmodule Reencodarr.Diagnostics do defp format_health_check_state(_), do: " Unknown state" - defp format_worker_sessions({:error, _}), do: " Unavailable" defp format_worker_sessions([]), do: " none" defp format_worker_sessions(worker_sessions) when is_list(worker_sessions) do @@ -669,8 +675,6 @@ defmodule Reencodarr.Diagnostics do end) end - defp format_worker_sessions(_), do: " Unknown state" - defp video_active_in_state?({:error, _}, _video_id), do: false defp video_active_in_state?(state, video_id) when is_map(state) do diff --git a/lib/reencodarr/media/video_queries.ex b/lib/reencodarr/media/video_queries.ex index c6239bd2..725cd769 100644 --- a/lib/reencodarr/media/video_queries.ex +++ b/lib/reencodarr/media/video_queries.ex @@ -193,7 +193,7 @@ defmodule Reencodarr.Media.VideoQueries do from(v in Video, where: v.state == :analyzed, order_by: [desc: v.priority, desc: v.bitrate, desc: v.size, asc: v.updated_at], - limit: 1, + limit: 10, select: v.id ) |> Repo.all(opts) diff --git a/lib/reencodarr_web/channels/worker_channel.ex b/lib/reencodarr_web/channels/worker_channel.ex index dc0ce490..08b6ff2e 100644 --- a/lib/reencodarr_web/channels/worker_channel.ex +++ b/lib/reencodarr_web/channels/worker_channel.ex @@ -6,6 +6,7 @@ defmodule ReencodarrWeb.WorkerChannel do use ReencodarrWeb, :channel alias Reencodarr.AbAv1.{WorkerProtocol, WorkerSessions} + alias Reencodarr.AbAv1.WorkerProtocol.Announcement alias Reencodarr.Media @crf_search_topic WorkerProtocol.crf_search_topic() @@ -15,18 +16,18 @@ defmodule ReencodarrWeb.WorkerChannel do {:ok, %{worker_id: worker_id}, socket} end - def join(_topic, _payload, socket), do: {:error, WorkerProtocol.error(:unauthorized), socket} + def join(_topic, _payload, _socket), do: {:error, WorkerProtocol.error(:unauthorized)} @impl true def handle_in("announce", payload, %{assigns: %{worker_id: server_worker_id}} = socket) do with {:ok, - %{ + %Announcement{ worker_id: client_worker_id, protocol_version: protocol_version, version: version, capabilities: capabilities }} <- WorkerProtocol.parse_announcement(payload), - true <- WorkerProtocol.supported_protocol_version?(protocol_version), + :ok <- ensure_supported_protocol_version(protocol_version), {:ok, _session} <- WorkerSessions.register(%{ server_worker_id: server_worker_id, @@ -44,9 +45,6 @@ defmodule ReencodarrWeb.WorkerChannel do {:reply, {:ok, WorkerProtocol.accepted(protocol_version)}, socket} else - false -> - {:reply, {:error, WorkerProtocol.error(:unsupported_protocol_version)}, socket} - {:error, reason} -> {:reply, {:error, WorkerProtocol.error(reason)}, socket} end @@ -66,6 +64,14 @@ defmodule ReencodarrWeb.WorkerChannel do def handle_in("pull_work", payload, socket), do: handle_work_request(payload, socket) + defp ensure_supported_protocol_version(protocol_version) do + if WorkerProtocol.supported_protocol_version?(protocol_version) do + :ok + else + {:error, :unsupported_protocol_version} + end + end + @impl true def terminate(_reason, %{assigns: %{worker_id: worker_id}} = socket) do maybe_requeue_active_video(socket) diff --git a/nix/dev-shell.nix b/nix/dev-shell.nix index bd1b58da..78146b4b 100644 --- a/nix/dev-shell.nix +++ b/nix/dev-shell.nix @@ -72,6 +72,11 @@ in export MIX_OS_DEPS_COMPILE_PARTITION_COUNT=$(( $(nproc) / 2 )) export ERL_AFLAGS="-kernel shell_history enabled" export SECRET_KEY_BASE="WEWsPGIpK/OgJA2ZcwzsgZxWKSAp35IsqWPYsvSUmm5awBUGpvsVOcG2kkDteXR1" + export REENCODARR_WORKER_TOKEN="$(printf '%s' "''${SECRET_KEY_BASE}:ab-av1-worker-token" \ + | ${pkgs.openssl}/bin/openssl dgst -sha256 -binary \ + | ${pkgs.coreutils}/bin/base64 -w 0 \ + | ${pkgs.coreutils}/bin/tr '+/' '-_' \ + | ${pkgs.coreutils}/bin/tr -d '=')" export COMPOSE_BAKE=true export GPG_TTY=$(tty) diff --git a/nix/module.nix b/nix/module.nix index caeb048e..2a20c339 100644 --- a/nix/module.nix +++ b/nix/module.nix @@ -30,6 +30,15 @@ ${lib.optionalString (cfg.secretKeyBaseFile != null) '' export SECRET_KEY_BASE="$(< "$CREDENTIALS_DIRECTORY/secret_key_base")" ''} + if [ -z "''${REENCODARR_WORKER_TOKEN:-}" ] && [ -n "''${SECRET_KEY_BASE:-}" ]; then + export REENCODARR_WORKER_TOKEN="$( + printf '%s' "''${SECRET_KEY_BASE}:ab-av1-worker-token" \ + | ${pkgs.openssl}/bin/openssl dgst -sha256 -binary \ + | ${pkgs.coreutils}/bin/base64 -w 0 \ + | ${pkgs.coreutils}/bin/tr '+/' '-_' \ + | ${pkgs.coreutils}/bin/tr -d '=' + )" + fi ''; startScript = pkgs.writeShellScript "reencodarr-start" '' diff --git a/test/reencodarr/ab_av1/worker_protocol_test.exs b/test/reencodarr/ab_av1/worker_protocol_test.exs index 39e61b2d..9e927827 100644 --- a/test/reencodarr/ab_av1/worker_protocol_test.exs +++ b/test/reencodarr/ab_av1/worker_protocol_test.exs @@ -2,11 +2,28 @@ defmodule Reencodarr.AbAv1.WorkerProtocolTest do use ExUnit.Case, async: true alias Reencodarr.AbAv1.WorkerProtocol + alias Reencodarr.AbAv1.WorkerProtocol.Announcement + + test "parses a worker announcement into a typed struct" do + assert {:ok, + %Announcement{ + worker_id: "abav1-dev", + protocol_version: 1, + version: "0.10.0", + capabilities: %{"crf_search" => true} + }} = + WorkerProtocol.parse_announcement(%{ + "worker_id" => "abav1-dev", + "protocol_version" => 1, + "version" => "0.10.0", + "capabilities" => %{"crf_search" => true} + }) + end test "builds a job_assigned payload from the claimed video" do payload = WorkerProtocol.work_assigned( - %{ + %Reencodarr.Media.Video{ id: 123, path: "/videos/movie.mkv", size: 987_654 diff --git a/test/reencodarr_web/channels/worker_channel_test.exs b/test/reencodarr_web/channels/worker_channel_test.exs index 383698fd..885bd476 100644 --- a/test/reencodarr_web/channels/worker_channel_test.exs +++ b/test/reencodarr_web/channels/worker_channel_test.exs @@ -4,6 +4,7 @@ defmodule ReencodarrWeb.WorkerChannelTest do alias Reencodarr.AbAv1.WorkerSessions alias Reencodarr.Fixtures alias Reencodarr.Media + alias ReencodarrWeb.WorkerChannel alias ReencodarrWeb.WorkerSocket describe "ab-av1 worker websocket" do @@ -216,6 +217,13 @@ defmodule ReencodarrWeb.WorkerChannelTest do after Application.delete_env(:reencodarr, :worker_token) end + + test "rejects invalid topics without crashing" do + assert {:error, %{reason: "unauthorized"}} = + WorkerChannel.join("workers:other", %{}, %{assigns: %{worker_id: "worker-1"}}) + after + Application.delete_env(:reencodarr, :worker_token) + end end defp announce_payload(overrides \\ []) do From 26019b698b8fe3846d3b988a20c784cd5ba19e9c Mon Sep 17 00:00:00 2001 From: Mika Cohen Date: Tue, 7 Jul 2026 11:55:20 -0600 Subject: [PATCH 15/70] Tighten worker websocket auth and sessions --- lib/reencodarr/ab_av1/worker_protocol.ex | 4 + lib/reencodarr/ab_av1/worker_sessions.ex | 114 +++++++++++++----- lib/reencodarr_web/channels/worker_channel.ex | 4 + lib/reencodarr_web/channels/worker_socket.ex | 13 +- .../ab_av1/worker_protocol_test.exs | 12 ++ .../ab_av1/worker_sessions_test.exs | 60 +++++---- .../live/dashboard_v2_live_test.exs | 2 +- 7 files changed, 141 insertions(+), 68 deletions(-) diff --git a/lib/reencodarr/ab_av1/worker_protocol.ex b/lib/reencodarr/ab_av1/worker_protocol.ex index 14a75941..419e3859 100644 --- a/lib/reencodarr/ab_av1/worker_protocol.ex +++ b/lib/reencodarr/ab_av1/worker_protocol.ex @@ -83,13 +83,16 @@ defmodule Reencodarr.AbAv1.WorkerProtocol do @spec error( :duplicate_worker_id | :invalid_announcement + | :invalid_session_attrs | :unsupported_protocol_version + | :unsupported_event | :unknown_worker_session | :unauthorized ) :: map() def error(:duplicate_worker_id), do: %{reason: "duplicate_worker_id"} def error(:invalid_announcement), do: %{reason: "invalid_announcement"} + def error(:invalid_session_attrs), do: %{reason: "invalid_session_attrs"} def error(:unsupported_protocol_version) do %{ @@ -98,6 +101,7 @@ defmodule Reencodarr.AbAv1.WorkerProtocol do } end + def error(:unsupported_event), do: %{reason: "unsupported_event"} def error(:unknown_worker_session), do: %{reason: "unknown_worker_session"} def error(:unauthorized), do: %{reason: "unauthorized"} end diff --git a/lib/reencodarr/ab_av1/worker_sessions.ex b/lib/reencodarr/ab_av1/worker_sessions.ex index a39301cd..646b7840 100644 --- a/lib/reencodarr/ab_av1/worker_sessions.ex +++ b/lib/reencodarr/ab_av1/worker_sessions.ex @@ -70,28 +70,24 @@ defmodule Reencodarr.AbAv1.WorkerSessions do @impl GenServer def handle_call({:register, attrs}, _from, state) do now = now() - server_worker_id = Map.fetch!(attrs, :server_worker_id) - client_worker_id = Map.fetch!(attrs, :client_worker_id) - case lookup_client(client_worker_id) do - nil -> - session = build_session(attrs, now) - :ok = put_session(session) - {:reply, {:ok, session}, state} - - ^server_worker_id -> - connected_at = lookup_session!(server_worker_id).connected_at - - session = - attrs - |> build_session(now) - |> Map.put(:connected_at, connected_at) - - :ok = put_session(session) - {:reply, {:ok, session}, state} - - _other_server_worker_id -> - {:reply, {:error, :duplicate_worker_id}, state} + with {:ok, server_worker_id} <- required_attr(attrs, :server_worker_id), + {:ok, client_worker_id} <- required_attr(attrs, :client_worker_id), + {:ok, version} <- required_attr(attrs, :version), + {:ok, protocol_version} <- required_attr(attrs, :protocol_version), + {:ok, capabilities} <- required_attr(attrs, :capabilities) do + register_session( + server_worker_id, + client_worker_id, + version, + protocol_version, + capabilities, + now, + state + ) + else + :error -> + {:reply, {:error, :invalid_session_attrs}, state} end end @@ -162,19 +158,78 @@ defmodule Reencodarr.AbAv1.WorkerSessions do {:noreply, state} end - defp build_session(attrs, now) do + defp build_session( + server_worker_id, + client_worker_id, + version, + protocol_version, + capabilities, + now + ) do %{ - server_worker_id: Map.fetch!(attrs, :server_worker_id), - client_worker_id: Map.fetch!(attrs, :client_worker_id), - version: Map.fetch!(attrs, :version), - protocol_version: Map.fetch!(attrs, :protocol_version), - capabilities: Map.fetch!(attrs, :capabilities), + server_worker_id: server_worker_id, + client_worker_id: client_worker_id, + version: version, + protocol_version: protocol_version, + capabilities: capabilities, active_video_id: nil, connected_at: now, last_seen_at: now } end + defp register_session( + server_worker_id, + client_worker_id, + version, + protocol_version, + capabilities, + now, + state + ) do + case lookup_session(server_worker_id) do + :error -> + case lookup_client(client_worker_id) do + nil -> + session = + build_session( + server_worker_id, + client_worker_id, + version, + protocol_version, + capabilities, + now + ) + + :ok = put_session(session) + {:reply, {:ok, session}, state} + + _existing_server_worker_id -> + {:reply, {:error, :duplicate_worker_id}, state} + end + + {:ok, %{client_worker_id: ^client_worker_id, connected_at: connected_at}} -> + session = + build_session( + server_worker_id, + client_worker_id, + version, + protocol_version, + capabilities, + now + ) + |> Map.put(:connected_at, connected_at) + + :ok = put_session(session) + {:reply, {:ok, session}, state} + + {:ok, _existing_session} -> + {:reply, {:error, :duplicate_worker_id}, state} + end + end + + defp required_attr(attrs, key), do: Map.fetch(attrs, key) + defp put_session(session) do :ok = drop_session(session.server_worker_id) true = :ets.insert(@by_server_table, {session.server_worker_id, session}) @@ -236,11 +291,6 @@ defmodule Reencodarr.AbAv1.WorkerSessions do end end - defp lookup_session!(server_worker_id) do - {:ok, session} = lookup_session(server_worker_id) - session - end - defp lookup_client(client_worker_id) do case :ets.lookup(@by_client_table, client_worker_id) do [{^client_worker_id, server_worker_id}] -> server_worker_id diff --git a/lib/reencodarr_web/channels/worker_channel.ex b/lib/reencodarr_web/channels/worker_channel.ex index 08b6ff2e..40c4d3c8 100644 --- a/lib/reencodarr_web/channels/worker_channel.ex +++ b/lib/reencodarr_web/channels/worker_channel.ex @@ -64,6 +64,10 @@ defmodule ReencodarrWeb.WorkerChannel do def handle_in("pull_work", payload, socket), do: handle_work_request(payload, socket) + def handle_in(_event, _payload, socket) do + {:reply, {:error, WorkerProtocol.error(:unsupported_event)}, socket} + end + defp ensure_supported_protocol_version(protocol_version) do if WorkerProtocol.supported_protocol_version?(protocol_version) do :ok diff --git a/lib/reencodarr_web/channels/worker_socket.ex b/lib/reencodarr_web/channels/worker_socket.ex index 94e64f43..839c4e08 100644 --- a/lib/reencodarr_web/channels/worker_socket.ex +++ b/lib/reencodarr_web/channels/worker_socket.ex @@ -10,10 +10,17 @@ defmodule ReencodarrWeb.WorkerSocket do channel WorkerProtocol.crf_search_topic(), ReencodarrWeb.WorkerChannel @impl true - def connect(%{"token" => token}, socket, _connect_info) do + def connect(%{"token" => token}, socket, _connect_info) when is_binary(token) do case Application.fetch_env(:reencodarr, :worker_token) do - {:ok, ^token} -> {:ok, assign(socket, :worker_id, worker_id())} - _ -> :error + {:ok, configured_token} when is_binary(configured_token) -> + if Plug.Crypto.secure_compare(configured_token, token) do + {:ok, assign(socket, :worker_id, worker_id())} + else + :error + end + + _ -> + :error end end diff --git a/test/reencodarr/ab_av1/worker_protocol_test.exs b/test/reencodarr/ab_av1/worker_protocol_test.exs index 9e927827..9150d343 100644 --- a/test/reencodarr/ab_av1/worker_protocol_test.exs +++ b/test/reencodarr/ab_av1/worker_protocol_test.exs @@ -4,6 +4,13 @@ defmodule Reencodarr.AbAv1.WorkerProtocolTest do alias Reencodarr.AbAv1.WorkerProtocol alias Reencodarr.AbAv1.WorkerProtocol.Announcement + test "rejects invalid announcement payloads" do + assert {:error, :invalid_announcement} = WorkerProtocol.parse_announcement(%{}) + + assert {:error, :invalid_announcement} = + WorkerProtocol.parse_announcement(%{"worker_id" => "x"}) + end + test "parses a worker announcement into a typed struct" do assert {:ok, %Announcement{ @@ -20,6 +27,11 @@ defmodule Reencodarr.AbAv1.WorkerProtocolTest do }) end + test "maps protocol errors to wire payloads" do + assert WorkerProtocol.error(:unauthorized) == %{reason: "unauthorized"} + assert WorkerProtocol.error(:unsupported_event) == %{reason: "unsupported_event"} + end + test "builds a job_assigned payload from the claimed video" do payload = WorkerProtocol.work_assigned( diff --git a/test/reencodarr/ab_av1/worker_sessions_test.exs b/test/reencodarr/ab_av1/worker_sessions_test.exs index b3fbd32f..950c0ad0 100644 --- a/test/reencodarr/ab_av1/worker_sessions_test.exs +++ b/test/reencodarr/ab_av1/worker_sessions_test.exs @@ -9,15 +9,21 @@ defmodule Reencodarr.AbAv1.WorkerSessionsTest do :ok end + test "rejects duplicate worker ids" do + assert {:ok, _session} = WorkerSessions.register(worker_session_attrs()) + + assert {:error, :duplicate_worker_id} = + WorkerSessions.register(worker_session_attrs(client_worker_id: "worker-client-2")) + end + + test "returns unknown worker session for missing ids" do + assert {:error, :unknown_worker_session} = WorkerSessions.touch("missing-worker") + assert {:error, :unknown_worker_session} = WorkerSessions.assign_video("missing-worker", 123) + assert {:error, :unknown_worker_session} = WorkerSessions.clear_video("missing-worker") + end + test "expires stale worker sessions" do - assert {:ok, _session} = - WorkerSessions.register(%{ - server_worker_id: "worker-server-1", - client_worker_id: "worker-client-1", - protocol_version: 1, - version: "0.10.0", - capabilities: %{"crf_search" => true} - }) + assert {:ok, _session} = WorkerSessions.register(worker_session_attrs()) assert [_session] = WorkerSessions.list() @@ -34,14 +40,7 @@ defmodule Reencodarr.AbAv1.WorkerSessionsTest do Application.put_env(:reencodarr, :worker_session_timeout_seconds, previous_timeout) end) - assert {:ok, _session} = - WorkerSessions.register(%{ - server_worker_id: "worker-server-1", - client_worker_id: "worker-client-1", - protocol_version: 1, - version: "0.10.0", - capabilities: %{"crf_search" => true} - }) + assert {:ok, _session} = WorkerSessions.register(worker_session_attrs()) send(Process.whereis(WorkerSessions), :expire_stale) :timer.sleep(25) @@ -50,14 +49,7 @@ defmodule Reencodarr.AbAv1.WorkerSessionsTest do end test "includes worker sessions in diagnostics output" do - assert {:ok, _session} = - WorkerSessions.register(%{ - server_worker_id: "worker-server-1", - client_worker_id: "worker-client-1", - protocol_version: 1, - version: "0.10.0", - capabilities: %{"crf_search" => true} - }) + assert {:ok, _session} = WorkerSessions.register(worker_session_attrs()) output = Diagnostics.processes() @@ -68,14 +60,7 @@ defmodule Reencodarr.AbAv1.WorkerSessionsTest do end test "tracks an assigned video on the session" do - assert {:ok, _session} = - WorkerSessions.register(%{ - server_worker_id: "worker-server-1", - client_worker_id: "worker-client-1", - protocol_version: 1, - version: "0.10.0", - capabilities: %{"crf_search" => true} - }) + assert {:ok, _session} = WorkerSessions.register(worker_session_attrs()) assert {:ok, session} = WorkerSessions.assign_video("worker-server-1", 123) assert session.active_video_id == 123 @@ -83,4 +68,15 @@ defmodule Reencodarr.AbAv1.WorkerSessionsTest do [listed_session] = WorkerSessions.list() assert listed_session.active_video_id == 123 end + + defp worker_session_attrs(overrides \\ []) do + %{ + server_worker_id: "worker-server-1", + client_worker_id: "worker-client-1", + protocol_version: 1, + version: "0.10.0", + capabilities: %{"crf_search" => true} + } + |> Map.merge(Map.new(overrides)) + end end diff --git a/test/reencodarr_web/live/dashboard_v2_live_test.exs b/test/reencodarr_web/live/dashboard_v2_live_test.exs index 1e4eb0ae..d683f422 100644 --- a/test/reencodarr_web/live/dashboard_v2_live_test.exs +++ b/test/reencodarr_web/live/dashboard_v2_live_test.exs @@ -7,7 +7,7 @@ defmodule ReencodarrWeb.DashboardLiveTest do - Button interactions without internal state checking - Event handling for service communication """ - use ReencodarrWeb.ConnCase, async: true + use ReencodarrWeb.ConnCase, async: false import Phoenix.LiveViewTest From 8c530ffa2ac6be99bdbd932797829238859922f1 Mon Sep 17 00:00:00 2001 From: Mika Cohen Date: Tue, 7 Jul 2026 12:04:57 -0600 Subject: [PATCH 16/70] Add CRF claim broadcast coverage --- test/reencodarr/broadway_producers_test.exs | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/test/reencodarr/broadway_producers_test.exs b/test/reencodarr/broadway_producers_test.exs index aebb723b..06388415 100644 --- a/test/reencodarr/broadway_producers_test.exs +++ b/test/reencodarr/broadway_producers_test.exs @@ -131,6 +131,25 @@ defmodule Reencodarr.BroadwayProducersTest do assert refreshed.state == :crf_searching end + test "atomic claim broadcasts video mutation for dashboard counts" do + Phoenix.PubSub.subscribe(Reencodarr.PubSub, "video_state_transitions") + expect_crf_available() + {:ok, video} = video_fixture(%{state: :analyzed}) + + state = init_state(CrfProducer) + {:noreply, [_claimed], _state2} = CrfProducer.handle_demand(1, state) + + assert_receive {:video_mutated, + %{ + action: :update, + old_video: %{id: old_id, state: :analyzed}, + new_video: %{id: new_id, state: :crf_searching} + }} + + assert old_id == video.id + assert new_id == video.id + end + test "claimed CRF videos are not dispatched again" do expect_crf_available() From 27c2e8f9c5346939a1b19ce9a4d13cfd37a2c35d Mon Sep 17 00:00:00 2001 From: Mika Cohen Date: Tue, 7 Jul 2026 12:32:01 -0600 Subject: [PATCH 17/70] Address worker websocket review feedback --- config/runtime.exs | 14 + lib/reencodarr/ab_av1/worker_protocol.ex | 409 ++++++++++++++++++ lib/reencodarr/ab_av1/worker_sessions.ex | 48 +- lib/reencodarr_web/channels/worker_channel.ex | 234 ++++++++++ .../ab_av1/worker_protocol_test.exs | 81 ++++ test/reencodarr/broadway_producers_test.exs | 19 + .../channels/worker_channel_test.exs | 199 +++++++++ 7 files changed, 977 insertions(+), 27 deletions(-) diff --git a/config/runtime.exs b/config/runtime.exs index 9734a4a0..85581309 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -22,6 +22,20 @@ end worker_token = System.get_env("REENCODARR_WORKER_TOKEN") + |> case do + nil -> + case System.get_env("SECRET_KEY_BASE") do + nil -> + nil + + secret_key_base -> + :crypto.mac(:hmac, :sha256, secret_key_base, "ab-av1-worker-token") + |> Base.url_encode64(padding: false) + end + + worker_token -> + worker_token + end if worker_token do config :reencodarr, :worker_token, worker_token diff --git a/lib/reencodarr/ab_av1/worker_protocol.ex b/lib/reencodarr/ab_av1/worker_protocol.ex index 419e3859..7a30885c 100644 --- a/lib/reencodarr/ab_av1/worker_protocol.ex +++ b/lib/reencodarr/ab_av1/worker_protocol.ex @@ -22,6 +22,51 @@ defmodule Reencodarr.AbAv1.WorkerProtocol do } end + @type video_id :: pos_integer() + @type percentage :: number() + @type crf_result :: %{ + required(:crf) => number(), + required(:score) => number(), + required(:percent) => number(), + optional(:size) => String.t() | nil, + optional(:time) => non_neg_integer() | nil, + optional(:params) => [String.t()], + optional(:target) => integer() | nil, + optional(:chosen) => boolean() + } + @type crf_search_progress :: %{ + required(:video_id) => video_id(), + required(:percent) => percentage(), + optional(:filename) => String.t() | nil, + optional(:eta) => non_neg_integer() | nil, + optional(:fps) => number() | nil + } + @type transfer_progress :: %{ + required(:video_id) => video_id(), + required(:transfer_id) => String.t(), + required(:percent) => percentage(), + required(:bytes_sent) => non_neg_integer(), + required(:total_bytes) => non_neg_integer(), + optional(:chunk_index) => non_neg_integer() | nil, + optional(:total_chunks) => non_neg_integer() | nil + } + @type failure_report :: %{ + required(:video_id) => video_id(), + required(:stage) => atom(), + required(:category) => atom(), + required(:message) => String.t(), + optional(:code) => String.t() | nil, + optional(:context) => map(), + optional(:retriable) => boolean() | nil, + optional(:stderr_excerpt) => String.t() | nil + } + @type completion_result :: :ok | :cancelled | :shutdown | {:error, term()} + @type crf_search_completed :: %{ + required(:video_id) => video_id(), + required(:result) => completion_result(), + optional(:chosen_crf) => number() | nil + } + @spec crf_search_topic() :: String.t() def crf_search_topic, do: @crf_search_topic @@ -56,9 +101,112 @@ defmodule Reencodarr.AbAv1.WorkerProtocol do def parse_announcement(_payload), do: {:error, :invalid_announcement} + @spec parse_transfer_progress(map()) :: {:ok, transfer_progress()} | {:error, atom()} + def parse_transfer_progress(payload) when is_map(payload) do + with {:ok, video_id} <- + required_integer(payload, [:video_id, "video_id"], :invalid_transfer_progress), + {:ok, transfer_id} <- + required_string(payload, [:transfer_id, "transfer_id"], :invalid_transfer_progress), + {:ok, percent} <- + required_number(payload, [:percent, "percent"], :invalid_transfer_progress) do + {:ok, + %{ + video_id: video_id, + transfer_id: transfer_id, + percent: percent, + bytes_sent: + optional_integer(payload, [ + :bytes_sent, + "bytes_sent", + :transferred_bytes, + "transferred_bytes" + ]) || 0, + total_bytes: optional_integer(payload, [:total_bytes, "total_bytes"]) || 0, + chunk_index: optional_integer(payload, [:chunk_index, "chunk_index"]), + total_chunks: optional_integer(payload, [:total_chunks, "total_chunks"]) + }} + end + end + + def parse_transfer_progress(_payload), do: {:error, :invalid_transfer_progress} + + @spec parse_crf_search_progress(map()) :: {:ok, crf_search_progress()} | {:error, atom()} + def parse_crf_search_progress(payload) when is_map(payload) do + with {:ok, video_id} <- + required_integer(payload, [:video_id, "video_id"], :invalid_crf_search_progress), + {:ok, percent} <- + required_number(payload, [:percent, "percent"], :invalid_crf_search_progress) do + {:ok, + %{ + video_id: video_id, + percent: percent, + filename: optional_string(payload, [:filename, "filename"]), + eta: optional_integer(payload, [:eta, "eta"]), + fps: optional_number(payload, [:fps, "fps"]) + }} + end + end + + def parse_crf_search_progress(_payload), do: {:error, :invalid_crf_search_progress} + + @spec parse_crf_search_result(map()) :: + {:ok, %{video_id: video_id(), results: [crf_result()]}} | {:error, atom()} + def parse_crf_search_result(payload) when is_map(payload) do + with {:ok, video_id} <- + required_integer(payload, [:video_id, "video_id"], :invalid_crf_search_result), + {:ok, results} <- parse_result_batch(payload) do + {:ok, %{video_id: video_id, results: results}} + end + end + + def parse_crf_search_result(_payload), do: {:error, :invalid_crf_search_result} + + @spec parse_failure_report(map()) :: {:ok, failure_report()} | {:error, atom()} + def parse_failure_report(payload) when is_map(payload) do + with {:ok, video_id} <- + required_integer(payload, [:video_id, "video_id"], :invalid_failure_report), + {:ok, stage} <- parse_failure_stage(payload), + {:ok, category} <- parse_failure_category(payload), + {:ok, message} <- + required_string(payload, [:message, "message"], :invalid_failure_report) do + {:ok, + %{ + video_id: video_id, + stage: stage, + category: category, + message: message, + code: optional_string(payload, [:code, "code"]), + context: optional_map(payload, [:context, "context"]) || %{}, + retriable: optional_boolean(payload, [:retriable, "retriable"]), + stderr_excerpt: optional_string(payload, [:stderr_excerpt, "stderr_excerpt"]) + }} + end + end + + def parse_failure_report(_payload), do: {:error, :invalid_failure_report} + + @spec parse_completion(map()) :: {:ok, crf_search_completed()} | {:error, atom()} + def parse_completion(payload) when is_map(payload) do + with {:ok, video_id} <- + required_integer(payload, [:video_id, "video_id"], :invalid_completion_result), + {:ok, result} <- parse_completion_result(payload) do + {:ok, + %{ + video_id: video_id, + result: result, + chosen_crf: optional_number(payload, [:chosen_crf, "chosen_crf"]) + }} + end + end + + def parse_completion(_payload), do: {:error, :invalid_completion_result} + @spec accepted(pos_integer()) :: map() def accepted(protocol_version), do: %{accepted: true, protocol_version: protocol_version} + @spec event_ack(String.t()) :: map() + def event_ack(event_name), do: %{accepted: true, event: event_name} + @spec no_work() :: map() def no_work, do: %{status: "no_work"} @@ -84,6 +232,11 @@ defmodule Reencodarr.AbAv1.WorkerProtocol do :duplicate_worker_id | :invalid_announcement | :invalid_session_attrs + | :invalid_transfer_progress + | :invalid_crf_search_progress + | :invalid_crf_search_result + | :invalid_failure_report + | :invalid_completion_result | :unsupported_protocol_version | :unsupported_event | :unknown_worker_session @@ -93,6 +246,11 @@ defmodule Reencodarr.AbAv1.WorkerProtocol do def error(:invalid_announcement), do: %{reason: "invalid_announcement"} def error(:invalid_session_attrs), do: %{reason: "invalid_session_attrs"} + def error(:invalid_transfer_progress), do: %{reason: "invalid_transfer_progress"} + def error(:invalid_crf_search_progress), do: %{reason: "invalid_crf_search_progress"} + def error(:invalid_crf_search_result), do: %{reason: "invalid_crf_search_result"} + def error(:invalid_failure_report), do: %{reason: "invalid_failure_report"} + def error(:invalid_completion_result), do: %{reason: "invalid_completion_result"} def error(:unsupported_protocol_version) do %{ @@ -104,4 +262,255 @@ defmodule Reencodarr.AbAv1.WorkerProtocol do def error(:unsupported_event), do: %{reason: "unsupported_event"} def error(:unknown_worker_session), do: %{reason: "unknown_worker_session"} def error(:unauthorized), do: %{reason: "unauthorized"} + + defp parse_single_crf_result(payload) when is_map(payload) do + with {:ok, crf} <- required_number(payload, [:crf, "crf"], :invalid_crf_search_result), + {:ok, score} <- + required_number( + payload, + [:score, "score", :vmaf_score, "vmaf_score"], + :invalid_crf_search_result + ), + {:ok, percent} <- + required_number( + payload, + [:percent, "percent", :vmaf_percentile, "vmaf_percentile"], + :invalid_crf_search_result + ) do + {:ok, + %{ + crf: crf, + score: score, + percent: percent, + size: optional_size(payload), + time: optional_time(payload), + params: optional_params(payload), + target: optional_integer(payload, [:target, "target"]), + chosen: optional_boolean(payload, [:chosen, "chosen"]) + }} + end + end + + defp parse_single_crf_result(_payload), do: {:error, :invalid_crf_search_result} + + defp parse_result_batch(payload) do + case fetch_any(payload, [:results, "results"]) do + nil -> + parse_single_crf_result(payload) + |> wrap_single_result() + + results when is_list(results) -> + parse_result_list(results) + + _ -> + {:error, :invalid_crf_search_result} + end + end + + defp parse_result_list(results) do + results + |> Enum.reduce_while({:ok, []}, fn result, {:ok, acc} -> + case parse_single_crf_result(result) do + {:ok, parsed_result} -> {:cont, {:ok, [parsed_result | acc]}} + {:error, _} = error -> {:halt, error} + end + end) + |> case do + {:ok, parsed_results} -> {:ok, Enum.reverse(parsed_results)} + {:error, _} = error -> error + end + end + + defp wrap_single_result({:ok, result}), do: {:ok, [result]} + defp wrap_single_result({:error, _} = error), do: error + + defp parse_completion_result(payload) do + case Map.get(payload, :result, Map.get(payload, "result")) do + value when value in [:ok, "ok"] -> {:ok, :ok} + value when value in [:cancelled, :canceled, "cancelled", "canceled"] -> {:ok, :cancelled} + value when value in [:shutdown, "shutdown"] -> {:ok, :shutdown} + value when value in [:failed, "failed"] -> {:ok, :failed} + {:error, _} = error -> {:ok, error} + %{"error" => reason} -> {:ok, {:error, reason}} + %{error: reason} -> {:ok, {:error, reason}} + _ -> {:error, :invalid_completion_result} + end + end + + defp parse_failure_stage(payload) do + case normalize_stage(Map.get(payload, :stage, Map.get(payload, "stage"))) do + {:ok, stage} -> {:ok, stage} + :error -> {:error, :invalid_failure_report} + end + end + + defp parse_failure_category(payload) do + case normalize_category(Map.get(payload, :category, Map.get(payload, "category"))) do + {:ok, category} -> {:ok, category} + :error -> {:error, :invalid_failure_report} + end + end + + defp normalize_stage(stage) when stage in [:analysis, :crf_search, :encoding, :post_process], + do: {:ok, stage} + + defp normalize_stage("analysis"), do: {:ok, :analysis} + defp normalize_stage("crf_search"), do: {:ok, :crf_search} + defp normalize_stage("encoding"), do: {:ok, :encoding} + defp normalize_stage("post_process"), do: {:ok, :post_process} + defp normalize_stage(_), do: :error + + defp normalize_category(category) + when category in [ + :file_access, + :mediainfo_parsing, + :validation, + :vmaf_calculation, + :crf_optimization, + :size_limits, + :preset_retry, + :process_failure, + :resource_exhaustion, + :codec_issues, + :timeout, + :file_operations, + :sync_integration, + :cleanup, + :configuration, + :system_environment, + :unknown + ], + do: {:ok, category} + + defp normalize_category("file_access"), do: {:ok, :file_access} + defp normalize_category("mediainfo_parsing"), do: {:ok, :mediainfo_parsing} + defp normalize_category("validation"), do: {:ok, :validation} + defp normalize_category("vmaf_calculation"), do: {:ok, :vmaf_calculation} + defp normalize_category("crf_optimization"), do: {:ok, :crf_optimization} + defp normalize_category("size_limits"), do: {:ok, :size_limits} + defp normalize_category("preset_retry"), do: {:ok, :preset_retry} + defp normalize_category("process_failure"), do: {:ok, :process_failure} + defp normalize_category("resource_exhaustion"), do: {:ok, :resource_exhaustion} + defp normalize_category("codec_issues"), do: {:ok, :codec_issues} + defp normalize_category("timeout"), do: {:ok, :timeout} + defp normalize_category("file_operations"), do: {:ok, :file_operations} + defp normalize_category("sync_integration"), do: {:ok, :sync_integration} + defp normalize_category("cleanup"), do: {:ok, :cleanup} + defp normalize_category("configuration"), do: {:ok, :configuration} + defp normalize_category("system_environment"), do: {:ok, :system_environment} + defp normalize_category("unknown"), do: {:ok, :unknown} + defp normalize_category(_), do: :error + + defp required_integer(payload, keys, error) do + case fetch_any(payload, keys) do + value when is_integer(value) and value > 0 -> {:ok, value} + _ -> {:error, error} + end + end + + defp required_number(payload, keys, error) do + case fetch_any(payload, keys) do + value when is_number(value) -> {:ok, value} + _ -> {:error, error} + end + end + + defp required_string(payload, keys, error) do + case fetch_any(payload, keys) do + value when is_binary(value) and value != "" -> {:ok, value} + _ -> {:error, error} + end + end + + defp optional_string(payload, keys) do + case fetch_any(payload, keys) do + value when is_binary(value) -> value + _ -> nil + end + end + + defp optional_number(payload, keys) do + case fetch_any(payload, keys) do + value when is_number(value) -> value + _ -> nil + end + end + + defp optional_integer(payload, keys) do + case fetch_any(payload, keys) do + value when is_integer(value) and value >= 0 -> value + _ -> nil + end + end + + defp optional_boolean(payload, keys) do + case fetch_any(payload, keys) do + value when is_boolean(value) -> value + _ -> nil + end + end + + defp optional_map(payload, keys) do + case fetch_any(payload, keys) do + value when is_map(value) -> value + _ -> nil + end + end + + defp optional_params(payload) do + case fetch_any(payload, [:params, "params"]) do + value when is_list(value) -> Enum.map(value, &to_string/1) + value when is_binary(value) -> [value] + _ -> [] + end + end + + defp optional_size(payload) do + case fetch_any(payload, [:size, "size"]) do + value when is_binary(value) -> value + value when is_number(value) -> format_size(value, payload) + _ -> optional_predicted_size(payload) + end + end + + defp format_size(size, payload) do + case optional_size_unit(payload) do + nil -> "#{size}" + unit -> "#{size} #{unit}" + end + end + + defp optional_predicted_size(payload) do + case optional_number(payload, [:predicted_size, "predicted_size"]) do + nil -> nil + size -> format_size(size, payload) + end + end + + defp optional_size_unit(payload) do + optional_string(payload, [:size_unit, "size_unit"]) || + optional_string(payload, [:unit, "unit"]) + end + + defp optional_time(payload) do + case fetch_any(payload, [:time, "time"]) do + value when is_integer(value) and value >= 0 -> + value + + value when is_number(value) -> + round(value) + + _ -> + case {optional_number(payload, [:time_taken, "time_taken"]), + optional_string(payload, [:time_unit, "time_unit"])} do + {nil, _} -> nil + {time_taken, unit} when unit in [nil, ""] -> round(time_taken) + {time_taken, unit} -> round(Reencodarr.Core.Time.to_seconds(time_taken, unit)) + end + end + end + + defp fetch_any(payload, keys) do + Enum.find_value(keys, fn key -> Map.get(payload, key) end) + end end diff --git a/lib/reencodarr/ab_av1/worker_sessions.ex b/lib/reencodarr/ab_av1/worker_sessions.ex index 646b7840..cadaeccb 100644 --- a/lib/reencodarr/ab_av1/worker_sessions.ex +++ b/lib/reencodarr/ab_av1/worker_sessions.ex @@ -92,15 +92,9 @@ defmodule Reencodarr.AbAv1.WorkerSessions do end def handle_call({:touch, server_worker_id}, _from, state) do - case lookup_session(server_worker_id) do - {:ok, session} -> - updated_session = %{session | last_seen_at: now()} - :ok = put_session(updated_session) - {:reply, {:ok, updated_session}, state} - - :error -> - {:reply, {:error, :unknown_worker_session}, state} - end + update_session_reply(server_worker_id, state, fn session -> + %{session | last_seen_at: now()} + end) end def handle_call({:unregister, server_worker_id}, _from, state) do @@ -109,27 +103,15 @@ defmodule Reencodarr.AbAv1.WorkerSessions do end def handle_call({:assign_video, server_worker_id, video_id}, _from, state) do - case lookup_session(server_worker_id) do - {:ok, session} -> - updated_session = %{session | active_video_id: video_id} - :ok = put_session(updated_session) - {:reply, {:ok, updated_session}, state} - - :error -> - {:reply, {:error, :unknown_worker_session}, state} - end + update_session_reply(server_worker_id, state, fn session -> + %{session | active_video_id: video_id} + end) end def handle_call({:clear_video, server_worker_id}, _from, state) do - case lookup_session(server_worker_id) do - {:ok, session} -> - updated_session = %{session | active_video_id: nil} - :ok = put_session(updated_session) - {:reply, {:ok, updated_session}, state} - - :error -> - {:reply, {:error, :unknown_worker_session}, state} - end + update_session_reply(server_worker_id, state, fn session -> + %{session | active_video_id: nil} + end) end def handle_call({:get, server_worker_id}, _from, state) do @@ -230,6 +212,18 @@ defmodule Reencodarr.AbAv1.WorkerSessions do defp required_attr(attrs, key), do: Map.fetch(attrs, key) + defp update_session_reply(server_worker_id, state, update_fun) do + case lookup_session(server_worker_id) do + {:ok, session} -> + updated_session = update_fun.(session) + :ok = put_session(updated_session) + {:reply, {:ok, updated_session}, state} + + :error -> + {:reply, {:error, :unknown_worker_session}, state} + end + end + defp put_session(session) do :ok = drop_session(session.server_worker_id) true = :ets.insert(@by_server_table, {session.server_worker_id, session}) diff --git a/lib/reencodarr_web/channels/worker_channel.ex b/lib/reencodarr_web/channels/worker_channel.ex index 40c4d3c8..f1059d83 100644 --- a/lib/reencodarr_web/channels/worker_channel.ex +++ b/lib/reencodarr_web/channels/worker_channel.ex @@ -7,6 +7,7 @@ defmodule ReencodarrWeb.WorkerChannel do alias Reencodarr.AbAv1.{WorkerProtocol, WorkerSessions} alias Reencodarr.AbAv1.WorkerProtocol.Announcement + alias Reencodarr.Dashboard.Events alias Reencodarr.Media @crf_search_topic WorkerProtocol.crf_search_topic() @@ -64,6 +65,21 @@ defmodule ReencodarrWeb.WorkerChannel do def handle_in("pull_work", payload, socket), do: handle_work_request(payload, socket) + def handle_in("transfer_progress", payload, socket), + do: handle_transfer_progress(payload, socket) + + def handle_in("crf_search_progress", payload, socket), + do: handle_crf_search_progress(payload, socket) + + def handle_in("crf_search_result", payload, socket), + do: handle_crf_search_result(payload, socket) + + def handle_in("crf_search_completed", payload, socket), + do: handle_crf_search_completed(payload, socket) + + def handle_in("video_failed", payload, socket), + do: handle_video_failed(payload, socket) + def handle_in(_event, _payload, socket) do {:reply, {:error, WorkerProtocol.error(:unsupported_event)}, socket} end @@ -101,6 +117,73 @@ defmodule ReencodarrWeb.WorkerChannel do end end + defp handle_transfer_progress(payload, %{assigns: %{worker_id: worker_id}} = socket) do + with {:ok, progress} <- WorkerProtocol.parse_transfer_progress(payload), + :ok <- ensure_active_video(socket, progress.video_id) do + Events.broadcast_event(:transfer_progress, Map.put(progress, :worker_id, worker_id)) + {:reply, {:ok, WorkerProtocol.event_ack("transfer_progress")}, socket} + else + {:error, reason} -> + {:reply, {:error, WorkerProtocol.error(reason)}, socket} + end + end + + defp handle_crf_search_progress(payload, socket) do + with {:ok, progress} <- WorkerProtocol.parse_crf_search_progress(payload), + :ok <- ensure_active_video(socket, progress.video_id) do + Events.broadcast_event(:crf_search_progress, progress) + {:reply, {:ok, WorkerProtocol.event_ack("crf_search_progress")}, socket} + else + {:error, reason} -> + {:reply, {:error, WorkerProtocol.error(reason)}, socket} + end + end + + defp handle_crf_search_result(payload, socket) do + with {:ok, result} <- WorkerProtocol.parse_crf_search_result(payload), + :ok <- ensure_active_video(socket, result.video_id) do + handle_valid_crf_search_result(socket, result) + else + {:error, reason} -> + {:reply, {:error, WorkerProtocol.error(reason)}, socket} + end + end + + defp handle_crf_search_completed(payload, %{assigns: %{worker_id: worker_id}} = socket) do + with {:ok, completion} <- WorkerProtocol.parse_completion(payload), + :ok <- ensure_active_video(socket, completion.video_id) do + handle_valid_crf_search_completion(worker_id, socket, completion) + else + {:error, reason} -> + {:reply, {:error, WorkerProtocol.error(reason)}, socket} + end + end + + defp handle_video_failed(payload, %{assigns: %{worker_id: worker_id}} = socket) do + with {:ok, failure} <- WorkerProtocol.parse_failure_report(payload), + :ok <- ensure_active_video(socket, failure.video_id) do + video = Media.get_video(failure.video_id) + + if is_nil(video) do + {:reply, {:error, WorkerProtocol.error(:unknown_worker_session)}, socket} + else + _ = + Media.record_video_failure(video, failure.stage, failure.category, + code: failure.code, + message: failure.message, + context: Map.put(failure.context, :stderr_excerpt, failure.stderr_excerpt) + ) + + socket = clear_assigned_video(worker_id, socket) + Events.broadcast_event(:video_failed, failure) + {:reply, {:ok, WorkerProtocol.event_ack("video_failed")}, socket} + end + else + {:error, reason} -> + {:reply, {:error, WorkerProtocol.error(reason)}, socket} + end + end + defp claim_work(worker_id, socket) do case Media.claim_next_video_for_crf_search() do %Media.Video{} = video -> @@ -145,4 +228,155 @@ defmodule ReencodarrWeb.WorkerChannel do end end end + + defp handle_valid_crf_search_result(socket, %{video_id: video_id, results: results}) do + case Media.get_video(video_id) do + nil -> + {:reply, {:error, WorkerProtocol.error(:unknown_worker_session)}, socket} + + video -> + persist_crf_results(video, results) + + if Enum.any?(results, &Map.get(&1, :chosen, false)) do + choose_result_from_report(video, results) + end + + {:reply, {:ok, WorkerProtocol.event_ack("crf_search_result")}, socket} + end + end + + defp handle_valid_crf_search_completion(worker_id, socket, completion) do + case Media.get_video(completion.video_id) do + nil -> + {:reply, {:error, WorkerProtocol.error(:unknown_worker_session)}, socket} + + video -> + socket = + apply_completion_result( + worker_id, + socket, + video, + completion.result, + completion.chosen_crf + ) + + Events.broadcast_event(:crf_search_completed, %{ + video_id: completion.video_id, + result: completion.result, + chosen_crf: completion.chosen_crf + }) + + {:reply, {:ok, WorkerProtocol.event_ack("crf_search_completed")}, socket} + end + end + + defp apply_completion_result(worker_id, socket, video, :ok, chosen_crf) do + finish_successful_crf_search(worker_id, socket, video, chosen_crf) + end + + defp apply_completion_result(worker_id, socket, video, :cancelled, _chosen_crf) do + finish_cancelled_crf_search(worker_id, socket, video, :cancelled) + end + + defp apply_completion_result(worker_id, socket, video, :shutdown, _chosen_crf) do + finish_cancelled_crf_search(worker_id, socket, video, :shutdown) + end + + defp apply_completion_result(worker_id, socket, video, :failed, _chosen_crf) do + record_completion_failure(worker_id, socket, video, "failed") + end + + defp apply_completion_result(worker_id, socket, video, {:error, reason}, _chosen_crf) do + record_completion_failure(worker_id, socket, video, inspect(reason)) + end + + defp record_completion_failure(worker_id, socket, video, code) do + _ = + Media.record_video_failure(video, :crf_search, :crf_optimization, + code: code, + message: "CRF search failed" + ) + + clear_assigned_video(worker_id, socket) + end + + defp ensure_active_video(socket, video_id) do + case socket.assigns[:current_video_id] do + nil -> {:error, :unknown_worker_session} + ^video_id -> :ok + _other -> {:error, :unknown_worker_session} + end + end + + defp persist_crf_results(video, results) do + Enum.each(results, fn result -> + attrs = + result + |> Map.put(:video_id, video.id) + |> Map.delete(:chosen) + + case Media.upsert_vmaf(attrs) do + {:ok, :skipped} -> :ok + {:ok, vmaf} -> Events.broadcast_event(:crf_search_vmaf_result, vmaf_to_event(vmaf)) + {:error, _} -> :ok + end + end) + end + + defp choose_result_from_report(video, results) do + chosen_crf = + Enum.find_value(results, fn result -> + if Map.get(result, :chosen, false), do: Map.get(result, :crf) + end) || Map.get(List.first(results) || %{}, :crf) + + case chosen_crf do + nil -> :ok + crf -> _ = Media.mark_vmaf_as_chosen(video.id, crf) + end + + _ = Media.mark_as_crf_searched(video) + _ = Media.resolve_crf_search_failures(video.id) + :ok + end + + defp finish_successful_crf_search(worker_id, socket, video, chosen_crf) do + case chosen_crf do + nil -> + case Media.choose_best_vmaf(video) do + {:ok, _vmaf} -> :ok + {:error, _} -> :ok + end + + crf -> + _ = Media.mark_vmaf_as_chosen(video.id, crf) + end + + _ = Media.mark_as_crf_searched(video) + _ = Media.resolve_crf_search_failures(video.id) + clear_assigned_video(worker_id, socket) + end + + defp finish_cancelled_crf_search(worker_id, socket, video, _reason) do + _ = Media.mark_as_analyzed(video) + clear_assigned_video(worker_id, socket) + end + + defp clear_assigned_video(worker_id, socket) do + _ = WorkerSessions.clear_video(worker_id) + + assign(socket, :current_video_id, nil) + |> assign(:current_vmaf_target, nil) + end + + defp vmaf_to_event(%Reencodarr.Media.Vmaf{} = vmaf) do + %{ + video_id: vmaf.video_id, + crf: vmaf.crf, + score: vmaf.score, + percent: vmaf.percent, + size: vmaf.size, + time: vmaf.time, + params: vmaf.params || [] + } + end end diff --git a/test/reencodarr/ab_av1/worker_protocol_test.exs b/test/reencodarr/ab_av1/worker_protocol_test.exs index 9150d343..dc0e0707 100644 --- a/test/reencodarr/ab_av1/worker_protocol_test.exs +++ b/test/reencodarr/ab_av1/worker_protocol_test.exs @@ -32,6 +32,87 @@ defmodule Reencodarr.AbAv1.WorkerProtocolTest do assert WorkerProtocol.error(:unsupported_event) == %{reason: "unsupported_event"} end + test "parses transfer progress into a typed payload" do + assert {:ok, + %{ + video_id: 123, + transfer_id: "transfer-1", + percent: 42.5, + bytes_sent: 1_048_576, + total_bytes: 2_097_152, + chunk_index: 3, + total_chunks: 8 + }} = + WorkerProtocol.parse_transfer_progress(%{ + "video_id" => 123, + "transfer_id" => "transfer-1", + "percent" => 42.5, + "bytes_sent" => 1_048_576, + "total_bytes" => 2_097_152, + "chunk_index" => 3, + "total_chunks" => 8 + }) + end + + test "parses structured CRF results and completion payloads" do + assert {:ok, + %{ + video_id: 123, + results: [ + %{ + crf: 28, + score: 95.4, + percent: 94, + size: "512 MB", + time: 3600, + params: ["--preset", "4"], + chosen: true + } + ] + }} = + WorkerProtocol.parse_crf_search_result(%{ + "video_id" => 123, + "crf" => 28, + "score" => 95.4, + "percent" => 94, + "size" => "512 MB", + "time" => 3600, + "params" => ["--preset", "4"], + "chosen" => true + }) + + assert {:ok, %{video_id: 123, result: :ok, chosen_crf: 28}} = + WorkerProtocol.parse_completion(%{ + "video_id" => 123, + "result" => "ok", + "chosen_crf" => 28 + }) + end + + test "parses typed failure reports" do + assert {:ok, + %{ + video_id: 123, + stage: :crf_search, + category: :timeout, + message: "timed out", + code: "EXIT_137", + context: %{node: "worker@host"}, + retriable: true, + stderr_excerpt: "ab-av1 timed out" + }} = + WorkerProtocol.parse_failure_report(%{ + "video_id" => 123, + "stage" => "crf_search", + "category" => "timeout", + "message" => "timed out", + "code" => "EXIT_137", + "context" => %{node: "worker@host"}, + "retriable" => true, + "stderr_excerpt" => "ab-av1 timed out" + }) + end + test "builds a job_assigned payload from the claimed video" do payload = WorkerProtocol.work_assigned( diff --git a/test/reencodarr/broadway_producers_test.exs b/test/reencodarr/broadway_producers_test.exs index 06388415..a05b89c1 100644 --- a/test/reencodarr/broadway_producers_test.exs +++ b/test/reencodarr/broadway_producers_test.exs @@ -150,6 +150,25 @@ defmodule Reencodarr.BroadwayProducersTest do assert new_id == video.id end + test "atomic claim broadcasts video mutation for CRF work" do + Phoenix.PubSub.subscribe(Reencodarr.PubSub, "video_state_transitions") + expect_crf_available() + {:ok, video} = video_fixture(%{state: :analyzed}) + + state = init_state(CrfProducer) + {:noreply, [_claimed], _state2} = CrfProducer.handle_demand(1, state) + + assert_receive {:video_mutated, + %{ + action: :update, + old_video: %{id: old_id, state: :analyzed}, + new_video: %{id: new_id, state: :crf_searching} + }} + + assert old_id == video.id + assert new_id == video.id + end + test "claimed CRF videos are not dispatched again" do expect_crf_available() diff --git a/test/reencodarr_web/channels/worker_channel_test.exs b/test/reencodarr_web/channels/worker_channel_test.exs index 885bd476..e05d20e5 100644 --- a/test/reencodarr_web/channels/worker_channel_test.exs +++ b/test/reencodarr_web/channels/worker_channel_test.exs @@ -2,6 +2,7 @@ defmodule ReencodarrWeb.WorkerChannelTest do use ReencodarrWeb.ChannelCase, async: false alias Reencodarr.AbAv1.WorkerSessions + alias Reencodarr.Dashboard.Events alias Reencodarr.Fixtures alias Reencodarr.Media alias ReencodarrWeb.WorkerChannel @@ -180,6 +181,204 @@ defmodule ReencodarrWeb.WorkerChannelTest do Application.delete_env(:reencodarr, :worker_token) end + test "broadcasts transfer and CRF progress updates" do + token = "test-worker-token" + Application.put_env(:reencodarr, :worker_token, token) + Phoenix.PubSub.subscribe(Reencodarr.PubSub, Events.channel()) + + {:ok, video} = Fixtures.video_fixture(%{state: :analyzed}) + video_id = video.id + + assert {:ok, socket} = connect(WorkerSocket, %{"token" => token}) + assert {:ok, _join_payload, socket} = subscribe_and_join(socket, "workers:crf_search") + + assert_reply push(socket, "announce", announce_payload(worker_id: "worker-a")), + :ok, + %{accepted: true, protocol_version: 1} + + assert_reply push(socket, "pull_work", %{}), + :ok, + %{status: "job_assigned", video_id: ^video_id} + + assert_reply push(socket, "transfer_progress", %{ + "video_id" => video_id, + "transfer_id" => "transfer-1", + "percent" => 25.5, + "bytes_sent" => 2_621_440, + "total_bytes" => 10_485_760, + "chunk_index" => 2, + "total_chunks" => 8 + }), + :ok, + %{accepted: true, event: "transfer_progress"} + + assert_receive {:transfer_progress, + %{ + video_id: ^video_id, + transfer_id: "transfer-1", + percent: 25.5, + bytes_sent: 2_621_440, + total_bytes: 10_485_760 + }} + + assert_reply push(socket, "crf_search_progress", %{ + "video_id" => video_id, + "percent" => 62.0, + "filename" => Path.basename(video.path), + "eta" => 90, + "fps" => 12.5 + }), + :ok, + %{accepted: true, event: "crf_search_progress"} + + assert_receive {:crf_search_progress, + %{ + video_id: ^video_id, + percent: 62.0, + filename: filename, + eta: 90, + fps: 12.5 + }} + + assert filename == Path.basename(video.path) + after + Application.delete_env(:reencodarr, :worker_token) + end + + test "submits structured CRF results and completes the job" do + token = "test-worker-token" + Application.put_env(:reencodarr, :worker_token, token) + Phoenix.PubSub.subscribe(Reencodarr.PubSub, Events.channel()) + + {:ok, video} = Fixtures.video_fixture(%{state: :analyzed}) + video_id = video.id + + assert {:ok, socket} = connect(WorkerSocket, %{"token" => token}) + assert {:ok, _join_payload, socket} = subscribe_and_join(socket, "workers:crf_search") + + assert_reply push(socket, "announce", announce_payload(worker_id: "worker-a")), + :ok, + %{accepted: true, protocol_version: 1} + + assert_reply push(socket, "pull_work", %{}), + :ok, + %{status: "job_assigned", video_id: ^video_id} + + assert_reply push(socket, "crf_search_result", %{ + "video_id" => video_id, + "results" => [ + %{ + "crf" => 26, + "score" => 94.1, + "percent" => 93.0, + "size" => "600 MB", + "params" => ["--preset", "4"] + }, + %{ + "crf" => 28, + "score" => 96.4, + "percent" => 95.0, + "size" => "520 MB", + "params" => ["--preset", "4"], + "chosen" => true + } + ] + }), + :ok, + %{accepted: true, event: "crf_search_result"} + + assert_receive {:crf_search_vmaf_result, %{video_id: ^video_id, crf: 26.0, score: 94.1}} + assert_receive {:crf_search_vmaf_result, %{video_id: ^video_id, crf: 28.0, score: 96.4}} + + assert_reply push(socket, "crf_search_completed", %{ + "video_id" => video_id, + "result" => "ok", + "chosen_crf" => 28 + }), + :ok, + %{accepted: true, event: "crf_search_completed"} + + assert_receive {:crf_search_completed, %{video_id: ^video_id, result: :ok, chosen_crf: 28}} + + assert Media.get_video(video_id).state == :crf_searched + assert Media.get_video(video_id).chosen_vmaf_id != nil + after + Application.delete_env(:reencodarr, :worker_token) + end + + test "records typed failures and cancels active work" do + token = "test-worker-token" + Application.put_env(:reencodarr, :worker_token, token) + Phoenix.PubSub.subscribe(Reencodarr.PubSub, Events.channel()) + + {:ok, video} = Fixtures.video_fixture(%{state: :analyzed}) + video_id = video.id + + assert {:ok, socket} = connect(WorkerSocket, %{"token" => token}) + assert {:ok, _join_payload, socket} = subscribe_and_join(socket, "workers:crf_search") + + assert_reply push(socket, "announce", announce_payload(worker_id: "worker-a")), + :ok, + %{accepted: true, protocol_version: 1} + + assert_reply push(socket, "pull_work", %{}), + :ok, + %{status: "job_assigned", video_id: ^video_id} + + assert_reply push(socket, "video_failed", %{ + "video_id" => video_id, + "stage" => "crf_search", + "category" => "timeout", + "message" => "timed out", + "code" => "EXIT_137", + "context" => %{"node" => "worker@host"}, + "retriable" => true, + "stderr_excerpt" => "ab-av1 timed out" + }), + :ok, + %{accepted: true, event: "video_failed"} + + assert_receive {:video_failed, + %{ + video_id: ^video_id, + stage: :crf_search, + category: :timeout, + message: "timed out" + }} + + assert Media.get_video(video_id).state == :failed + assert [_failure] = Media.get_video_failures(video_id) + + {:ok, cancelled_video} = Fixtures.video_fixture(%{state: :analyzed}) + cancelled_video_id = cancelled_video.id + + assert {:ok, cancel_socket} = connect(WorkerSocket, %{"token" => token}) + + assert {:ok, _join_payload, cancel_socket} = + subscribe_and_join(cancel_socket, "workers:crf_search") + + assert_reply push(cancel_socket, "announce", announce_payload(worker_id: "worker-b")), + :ok, + %{accepted: true, protocol_version: 1} + + assert_reply push(cancel_socket, "pull_work", %{}), + :ok, + %{status: "job_assigned", video_id: ^cancelled_video_id} + + assert_reply push(cancel_socket, "crf_search_completed", %{ + "video_id" => cancelled_video_id, + "result" => "cancelled" + }), + :ok, + %{accepted: true, event: "crf_search_completed"} + + assert_receive {:crf_search_completed, %{video_id: ^cancelled_video_id, result: :cancelled}} + + assert Media.get_video(cancelled_video_id).state == :analyzed + after + Application.delete_env(:reencodarr, :worker_token) + end + test "requeues active work when the worker disconnects" do token = "test-worker-token" Application.put_env(:reencodarr, :worker_token, token) From dac20fa8206481cd46a08b6b3416962b76c8763b Mon Sep 17 00:00:00 2001 From: Mika Cohen Date: Tue, 7 Jul 2026 12:39:28 -0600 Subject: [PATCH 18/70] Tighten worker websocket protocol types --- lib/reencodarr/ab_av1/worker_protocol.ex | 144 +++++++++++++----- .../crf_searcher/broadway/producer.ex | 11 +- .../ab_av1/worker_protocol_test.exs | 31 +++- 3 files changed, 134 insertions(+), 52 deletions(-) diff --git a/lib/reencodarr/ab_av1/worker_protocol.ex b/lib/reencodarr/ab_av1/worker_protocol.ex index 7a30885c..8b88edce 100644 --- a/lib/reencodarr/ab_av1/worker_protocol.ex +++ b/lib/reencodarr/ab_av1/worker_protocol.ex @@ -22,6 +22,98 @@ defmodule Reencodarr.AbAv1.WorkerProtocol do } end + defmodule TransferProgress do + @moduledoc false + + @enforce_keys [:video_id, :transfer_id, :percent, :bytes_sent, :total_bytes] + defstruct [ + :video_id, + :transfer_id, + :percent, + :bytes_sent, + :total_bytes, + :chunk_index, + :total_chunks + ] + + @type t :: %__MODULE__{ + video_id: pos_integer(), + transfer_id: String.t(), + percent: number(), + bytes_sent: non_neg_integer(), + total_bytes: non_neg_integer(), + chunk_index: non_neg_integer() | nil, + total_chunks: non_neg_integer() | nil + } + end + + defmodule CrfSearchProgress do + @moduledoc false + + @enforce_keys [:video_id, :percent] + defstruct [:video_id, :percent, :filename, :eta, :fps] + + @type t :: %__MODULE__{ + video_id: pos_integer(), + percent: number(), + filename: String.t() | nil, + eta: non_neg_integer() | nil, + fps: number() | nil + } + end + + defmodule CrfSearchResult do + @moduledoc false + + @enforce_keys [:video_id, :results] + defstruct [:video_id, :results] + + @type t :: %__MODULE__{ + video_id: pos_integer(), + results: [map()] + } + end + + defmodule FailureReport do + @moduledoc false + + @enforce_keys [:video_id, :stage, :category, :message] + defstruct [ + :video_id, + :stage, + :category, + :message, + :code, + :context, + :retriable, + :stderr_excerpt + ] + + @type t :: %__MODULE__{ + video_id: pos_integer(), + stage: atom(), + category: atom(), + message: String.t(), + code: String.t() | nil, + context: map(), + retriable: boolean() | nil, + stderr_excerpt: String.t() | nil + } + end + + defmodule Completion do + @moduledoc false + + @enforce_keys [:video_id, :result] + defstruct [:video_id, :result, :chosen_crf] + + @type t :: %__MODULE__{ + video_id: pos_integer(), + result: :ok | :cancelled | :shutdown | {:error, term()}, + chosen_crf: number() | nil + } + end + @type video_id :: pos_integer() @type percentage :: number() @type crf_result :: %{ @@ -34,38 +126,7 @@ defmodule Reencodarr.AbAv1.WorkerProtocol do optional(:target) => integer() | nil, optional(:chosen) => boolean() } - @type crf_search_progress :: %{ - required(:video_id) => video_id(), - required(:percent) => percentage(), - optional(:filename) => String.t() | nil, - optional(:eta) => non_neg_integer() | nil, - optional(:fps) => number() | nil - } - @type transfer_progress :: %{ - required(:video_id) => video_id(), - required(:transfer_id) => String.t(), - required(:percent) => percentage(), - required(:bytes_sent) => non_neg_integer(), - required(:total_bytes) => non_neg_integer(), - optional(:chunk_index) => non_neg_integer() | nil, - optional(:total_chunks) => non_neg_integer() | nil - } - @type failure_report :: %{ - required(:video_id) => video_id(), - required(:stage) => atom(), - required(:category) => atom(), - required(:message) => String.t(), - optional(:code) => String.t() | nil, - optional(:context) => map(), - optional(:retriable) => boolean() | nil, - optional(:stderr_excerpt) => String.t() | nil - } @type completion_result :: :ok | :cancelled | :shutdown | {:error, term()} - @type crf_search_completed :: %{ - required(:video_id) => video_id(), - required(:result) => completion_result(), - optional(:chosen_crf) => number() | nil - } @spec crf_search_topic() :: String.t() def crf_search_topic, do: @crf_search_topic @@ -101,7 +162,7 @@ defmodule Reencodarr.AbAv1.WorkerProtocol do def parse_announcement(_payload), do: {:error, :invalid_announcement} - @spec parse_transfer_progress(map()) :: {:ok, transfer_progress()} | {:error, atom()} + @spec parse_transfer_progress(map()) :: {:ok, TransferProgress.t()} | {:error, atom()} def parse_transfer_progress(payload) when is_map(payload) do with {:ok, video_id} <- required_integer(payload, [:video_id, "video_id"], :invalid_transfer_progress), @@ -110,7 +171,7 @@ defmodule Reencodarr.AbAv1.WorkerProtocol do {:ok, percent} <- required_number(payload, [:percent, "percent"], :invalid_transfer_progress) do {:ok, - %{ + %TransferProgress{ video_id: video_id, transfer_id: transfer_id, percent: percent, @@ -130,14 +191,14 @@ defmodule Reencodarr.AbAv1.WorkerProtocol do def parse_transfer_progress(_payload), do: {:error, :invalid_transfer_progress} - @spec parse_crf_search_progress(map()) :: {:ok, crf_search_progress()} | {:error, atom()} + @spec parse_crf_search_progress(map()) :: {:ok, CrfSearchProgress.t()} | {:error, atom()} def parse_crf_search_progress(payload) when is_map(payload) do with {:ok, video_id} <- required_integer(payload, [:video_id, "video_id"], :invalid_crf_search_progress), {:ok, percent} <- required_number(payload, [:percent, "percent"], :invalid_crf_search_progress) do {:ok, - %{ + %CrfSearchProgress{ video_id: video_id, percent: percent, filename: optional_string(payload, [:filename, "filename"]), @@ -149,19 +210,18 @@ defmodule Reencodarr.AbAv1.WorkerProtocol do def parse_crf_search_progress(_payload), do: {:error, :invalid_crf_search_progress} - @spec parse_crf_search_result(map()) :: - {:ok, %{video_id: video_id(), results: [crf_result()]}} | {:error, atom()} + @spec parse_crf_search_result(map()) :: {:ok, CrfSearchResult.t()} | {:error, atom()} def parse_crf_search_result(payload) when is_map(payload) do with {:ok, video_id} <- required_integer(payload, [:video_id, "video_id"], :invalid_crf_search_result), {:ok, results} <- parse_result_batch(payload) do - {:ok, %{video_id: video_id, results: results}} + {:ok, %CrfSearchResult{video_id: video_id, results: results}} end end def parse_crf_search_result(_payload), do: {:error, :invalid_crf_search_result} - @spec parse_failure_report(map()) :: {:ok, failure_report()} | {:error, atom()} + @spec parse_failure_report(map()) :: {:ok, FailureReport.t()} | {:error, atom()} def parse_failure_report(payload) when is_map(payload) do with {:ok, video_id} <- required_integer(payload, [:video_id, "video_id"], :invalid_failure_report), @@ -170,7 +230,7 @@ defmodule Reencodarr.AbAv1.WorkerProtocol do {:ok, message} <- required_string(payload, [:message, "message"], :invalid_failure_report) do {:ok, - %{ + %FailureReport{ video_id: video_id, stage: stage, category: category, @@ -185,13 +245,13 @@ defmodule Reencodarr.AbAv1.WorkerProtocol do def parse_failure_report(_payload), do: {:error, :invalid_failure_report} - @spec parse_completion(map()) :: {:ok, crf_search_completed()} | {:error, atom()} + @spec parse_completion(map()) :: {:ok, Completion.t()} | {:error, atom()} def parse_completion(payload) when is_map(payload) do with {:ok, video_id} <- required_integer(payload, [:video_id, "video_id"], :invalid_completion_result), {:ok, result} <- parse_completion_result(payload) do {:ok, - %{ + %Completion{ video_id: video_id, result: result, chosen_crf: optional_number(payload, [:chosen_crf, "chosen_crf"]) diff --git a/lib/reencodarr/crf_searcher/broadway/producer.ex b/lib/reencodarr/crf_searcher/broadway/producer.ex index e21a21e0..805db2a4 100644 --- a/lib/reencodarr/crf_searcher/broadway/producer.ex +++ b/lib/reencodarr/crf_searcher/broadway/producer.ex @@ -63,13 +63,12 @@ defmodule Reencodarr.CrfSearcher.Broadway.Producer do suspended? = ProcessControl.suspended?(:crf_searcher) videos = - if status == :available and not suspended? do - case Media.claim_next_video_for_crf_search() do - nil -> [] - video -> [video] - end + with :available <- status, + false <- suspended?, + video when not is_nil(video) <- Media.claim_next_video_for_crf_search() do + [video] else - [] + _ -> [] end remaining_demand = demand - length(videos) diff --git a/test/reencodarr/ab_av1/worker_protocol_test.exs b/test/reencodarr/ab_av1/worker_protocol_test.exs index dc0e0707..8227fe03 100644 --- a/test/reencodarr/ab_av1/worker_protocol_test.exs +++ b/test/reencodarr/ab_av1/worker_protocol_test.exs @@ -3,6 +3,11 @@ defmodule Reencodarr.AbAv1.WorkerProtocolTest do alias Reencodarr.AbAv1.WorkerProtocol alias Reencodarr.AbAv1.WorkerProtocol.Announcement + alias Reencodarr.AbAv1.WorkerProtocol.Completion + alias Reencodarr.AbAv1.WorkerProtocol.CrfSearchProgress + alias Reencodarr.AbAv1.WorkerProtocol.CrfSearchResult + alias Reencodarr.AbAv1.WorkerProtocol.FailureReport + alias Reencodarr.AbAv1.WorkerProtocol.TransferProgress test "rejects invalid announcement payloads" do assert {:error, :invalid_announcement} = WorkerProtocol.parse_announcement(%{}) @@ -34,7 +39,7 @@ defmodule Reencodarr.AbAv1.WorkerProtocolTest do test "parses transfer progress into a typed payload" do assert {:ok, - %{ + %TransferProgress{ video_id: 123, transfer_id: "transfer-1", percent: 42.5, @@ -54,9 +59,27 @@ defmodule Reencodarr.AbAv1.WorkerProtocolTest do }) end + test "parses CRF search progress into a typed payload" do + assert {:ok, + %CrfSearchProgress{ + video_id: 123, + percent: 78.5, + filename: "movie.mkv", + eta: 42, + fps: 23.97 + }} = + WorkerProtocol.parse_crf_search_progress(%{ + "video_id" => 123, + "percent" => 78.5, + "filename" => "movie.mkv", + "eta" => 42, + "fps" => 23.97 + }) + end + test "parses structured CRF results and completion payloads" do assert {:ok, - %{ + %CrfSearchResult{ video_id: 123, results: [ %{ @@ -81,7 +104,7 @@ defmodule Reencodarr.AbAv1.WorkerProtocolTest do "chosen" => true }) - assert {:ok, %{video_id: 123, result: :ok, chosen_crf: 28}} = + assert {:ok, %Completion{video_id: 123, result: :ok, chosen_crf: 28}} = WorkerProtocol.parse_completion(%{ "video_id" => 123, "result" => "ok", @@ -91,7 +114,7 @@ defmodule Reencodarr.AbAv1.WorkerProtocolTest do test "parses typed failure reports" do assert {:ok, - %{ + %FailureReport{ video_id: 123, stage: :crf_search, category: :timeout, From 526d81805da1d6966342ecc29e9cd4170e85d057 Mon Sep 17 00:00:00 2001 From: Mika Cohen Date: Tue, 7 Jul 2026 12:52:48 -0600 Subject: [PATCH 19/70] Add workers status live view --- lib/reencodarr/ab_av1/worker_sessions.ex | 14 ++ lib/reencodarr_web/live/dashboard_live.ex | 10 ++ lib/reencodarr_web/live/workers_live.ex | 138 ++++++++++++++++++ lib/reencodarr_web/router.ex | 1 + .../live/dashboard_v2_live_test.exs | 1 + .../reencodarr_web/live/workers_live_test.exs | 40 +++++ 6 files changed, 204 insertions(+) create mode 100644 lib/reencodarr_web/live/workers_live.ex create mode 100644 test/reencodarr_web/live/workers_live_test.exs diff --git a/lib/reencodarr/ab_av1/worker_sessions.ex b/lib/reencodarr/ab_av1/worker_sessions.ex index cadaeccb..274d3806 100644 --- a/lib/reencodarr/ab_av1/worker_sessions.ex +++ b/lib/reencodarr/ab_av1/worker_sessions.ex @@ -5,6 +5,8 @@ defmodule Reencodarr.AbAv1.WorkerSessions do use GenServer + alias Reencodarr.Dashboard.Events + @by_server_table :reencodarr_worker_sessions_by_server @by_client_table :reencodarr_worker_sessions_by_client @@ -99,6 +101,7 @@ defmodule Reencodarr.AbAv1.WorkerSessions do def handle_call({:unregister, server_worker_id}, _from, state) do :ok = drop_session(server_worker_id) + broadcast_sessions() {:reply, :ok, state} end @@ -120,6 +123,11 @@ defmodule Reencodarr.AbAv1.WorkerSessions do def handle_call({:expire_stale, timeout_seconds}, _from, state) do {expired_sessions, _} = expire_stale_sessions(timeout_seconds) + + if expired_sessions != [] do + broadcast_sessions() + end + {:reply, {:ok, expired_sessions}, state} end @@ -130,6 +138,7 @@ defmodule Reencodarr.AbAv1.WorkerSessions do def handle_call(:reset, _from, state) do :ok = reset_tables() + broadcast_sessions() {:reply, :ok, state} end @@ -224,10 +233,15 @@ defmodule Reencodarr.AbAv1.WorkerSessions do end end + defp broadcast_sessions do + Events.broadcast_event(:worker_sessions_updated, %{sessions: list_sessions()}) + end + defp put_session(session) do :ok = drop_session(session.server_worker_id) true = :ets.insert(@by_server_table, {session.server_worker_id, session}) true = :ets.insert(@by_client_table, {session.client_worker_id, session.server_worker_id}) + broadcast_sessions() :ok end diff --git a/lib/reencodarr_web/live/dashboard_live.ex b/lib/reencodarr_web/live/dashboard_live.ex index c82c029a..e9313bdf 100644 --- a/lib/reencodarr_web/live/dashboard_live.ex +++ b/lib/reencodarr_web/live/dashboard_live.ex @@ -1060,6 +1060,16 @@ defmodule ReencodarrWeb.DashboardLive do <% end %>
+ +
+
Worker sessions
+ <.link + navigate={~p"/workers"} + class="rounded-full border border-cyan-900 bg-cyan-950 px-3 py-1 text-xs text-cyan-300 hover:border-cyan-700 hover:text-white" + > + Open workers + +
diff --git a/lib/reencodarr_web/live/workers_live.ex b/lib/reencodarr_web/live/workers_live.ex new file mode 100644 index 00000000..42fd5b10 --- /dev/null +++ b/lib/reencodarr_web/live/workers_live.ex @@ -0,0 +1,138 @@ +defmodule ReencodarrWeb.WorkersLive do + @moduledoc """ + LiveView for connected ab-av1 workers and their current status. + """ + + use ReencodarrWeb, :live_view + + alias Reencodarr.AbAv1.WorkerSessions + alias Reencodarr.Dashboard.Events + + @refresh_interval 5_000 + + @impl true + def mount(_params, _session, socket) do + socket = assign(socket, :workers, WorkerSessions.list()) + + if connected?(socket) do + Phoenix.PubSub.subscribe(Reencodarr.PubSub, Events.channel()) + Process.send_after(self(), :refresh_workers, @refresh_interval) + end + + {:ok, socket} + end + + @impl true + def handle_info(:refresh_workers, socket) do + Process.send_after(self(), :refresh_workers, @refresh_interval) + {:noreply, assign(socket, :workers, WorkerSessions.list())} + end + + @impl true + def handle_info({:worker_sessions_updated, %{sessions: sessions}}, socket) do + {:noreply, assign(socket, :workers, sessions)} + end + + def handle_info({_event, _data}, socket), do: {:noreply, socket} + + @impl true + def render(assigns) do + ~H""" +
+
+
+
+

ab-av1

+

Workers

+

+ Connected worker sessions and their current job state. +

+
+ +
+
+ Connected {length(@workers)} +
+ <.link + navigate={~p"/"} + class="rounded-full border border-gray-800 bg-gray-900 px-3 py-1 text-xs text-gray-300 hover:border-cyan-700 hover:text-white" + > + Dashboard + +
+
+ +
+ + + + + + + + + + + + + + <%= if @workers == [] do %> + + + + <% end %> + + <%= for worker <- @workers do %> + + + + + + + + + + <% end %> + +
WorkerStatusVideoProtocolVersionConnectedLast Seen
+ No workers connected. +
+
{worker.client_worker_id}
+
+ server: {worker.server_worker_id} +
+ 		+ + {worker_status(worker)} + + + {worker_video(worker)} + + {worker.protocol_version} + + {worker.version} + + {DateTime.to_iso8601(worker.connected_at)} + + {DateTime.to_iso8601(worker.last_seen_at)} +
+
+
+
+ """ + end + + defp worker_status(%{active_video_id: nil}), do: "Idle" + defp worker_status(%{active_video_id: _video_id}), do: "Working" + + defp worker_video(%{active_video_id: nil}), do: "none" + defp worker_video(%{active_video_id: video_id}), do: "video ##{video_id}" + + defp status_badge_class(%{active_video_id: nil}), + do: + "inline-flex rounded-full border border-emerald-900 bg-emerald-950 px-2 py-1 text-xs font-medium text-emerald-300" + + defp status_badge_class(%{active_video_id: _video_id}), + do: + "inline-flex rounded-full border border-cyan-900 bg-cyan-950 px-2 py-1 text-xs font-medium text-cyan-300" +end diff --git a/lib/reencodarr_web/router.ex b/lib/reencodarr_web/router.ex index aa2840e6..94050d76 100644 --- a/lib/reencodarr_web/router.ex +++ b/lib/reencodarr_web/router.ex @@ -30,6 +30,7 @@ defmodule ReencodarrWeb.Router do live_session :default, layout: {ReencodarrWeb.Layouts, :app} do live "/", DashboardLive, :index + live "/workers", WorkersLive, :index live "/broadway", BroadwayLive, :index live "/bad-files", BadFilesLive, :index live "/failures", FailuresLive, :index diff --git a/test/reencodarr_web/live/dashboard_v2_live_test.exs b/test/reencodarr_web/live/dashboard_v2_live_test.exs index d683f422..6f7195d6 100644 --- a/test/reencodarr_web/live/dashboard_v2_live_test.exs +++ b/test/reencodarr_web/live/dashboard_v2_live_test.exs @@ -24,6 +24,7 @@ defmodule ReencodarrWeb.DashboardLiveTest do assert html =~ "Media Library Sync" assert html =~ "Sonarr" assert html =~ "Radarr" + assert html =~ "Open workers" assert html =~ ~s(id="dashboard-root") assert html =~ ~s(phx-hook="DashboardAnimations") assert html =~ ~s(id="dashboard-active-work") diff --git a/test/reencodarr_web/live/workers_live_test.exs b/test/reencodarr_web/live/workers_live_test.exs new file mode 100644 index 00000000..786c83f0 --- /dev/null +++ b/test/reencodarr_web/live/workers_live_test.exs @@ -0,0 +1,40 @@ +defmodule ReencodarrWeb.WorkersLiveTest do + use ReencodarrWeb.ConnCase, async: false + + import Phoenix.LiveViewTest + + alias Reencodarr.AbAv1.WorkerSessions + + setup do + WorkerSessions.reset() + :ok + end + + test "renders an empty state when no workers are connected", %{conn: conn} do + {:ok, _view, html} = live(conn, ~p"/workers") + + assert html =~ "Workers" + assert html =~ "No workers connected." + end + + test "renders connected worker details and refreshes on session updates", %{conn: conn} do + {:ok, _session} = + WorkerSessions.register(%{ + server_worker_id: "worker-server-1", + client_worker_id: "worker-client-1", + protocol_version: 1, + version: "0.10.0", + capabilities: %{"crf_search" => true} + }) + + assert {:ok, view, html} = live(conn, ~p"/workers") + assert html =~ "worker-client-1" + assert html =~ "Idle" + + assert {:ok, _session} = WorkerSessions.assign_video("worker-server-1", 123) + + html = render(view) + assert html =~ "Working" + assert html =~ "video #123" + end +end From 367a2cef268a7dc247f4868ec5e80adc45b8d99c Mon Sep 17 00:00:00 2001 From: Mika Cohen Date: Tue, 7 Jul 2026 13:00:46 -0600 Subject: [PATCH 20/70] Wire workers into main navigation --- lib/reencodarr_web/components/layouts/app.html.heex | 12 ++++++++++++ test/reencodarr_web/live/workers_live_test.exs | 1 + 2 files changed, 13 insertions(+) diff --git a/lib/reencodarr_web/components/layouts/app.html.heex b/lib/reencodarr_web/components/layouts/app.html.heex index fda0a927..75a5f76b 100644 --- a/lib/reencodarr_web/components/layouts/app.html.heex +++ b/lib/reencodarr_web/components/layouts/app.html.heex @@ -48,6 +48,12 @@ > Failures + <.link + navigate={~p"/workers"} + class="rounded px-3 py-2 text-sm text-gray-300 transition-colors hover:bg-gray-800 hover:text-white" + > + Workers + <.link navigate={~p"/videos"} class="rounded px-3 py-2 text-sm text-gray-300 transition-colors hover:bg-gray-800 hover:text-white" @@ -94,6 +100,12 @@ > Failures + <.link + navigate={~p"/workers"} + class="rounded px-2 py-1 text-xs text-gray-300 transition-colors hover:bg-gray-700 hover:text-white sm:px-3 sm:py-2 sm:text-sm" + > + Workers + <.link navigate={~p"/videos"} class="rounded px-2 py-1 text-xs text-gray-300 transition-colors hover:bg-gray-700 hover:text-white sm:px-3 sm:py-2 sm:text-sm" diff --git a/test/reencodarr_web/live/workers_live_test.exs b/test/reencodarr_web/live/workers_live_test.exs index 786c83f0..3f4dbb11 100644 --- a/test/reencodarr_web/live/workers_live_test.exs +++ b/test/reencodarr_web/live/workers_live_test.exs @@ -15,6 +15,7 @@ defmodule ReencodarrWeb.WorkersLiveTest do assert html =~ "Workers" assert html =~ "No workers connected." + assert html =~ ~s(href="/workers") end test "renders connected worker details and refreshes on session updates", %{conn: conn} do From 9f12a0c6f67348d50112cdeca5198f080a5cde0b Mon Sep 17 00:00:00 2001 From: Mika Cohen Date: Tue, 7 Jul 2026 13:22:03 -0600 Subject: [PATCH 21/70] Tighten CRF claim query --- lib/reencodarr/media/video_queries.ex | 11 ++++------- test/reencodarr/broadway_producers_test.exs | 19 ------------------- 2 files changed, 4 insertions(+), 26 deletions(-) diff --git a/lib/reencodarr/media/video_queries.ex b/lib/reencodarr/media/video_queries.ex index 725cd769..e907c71b 100644 --- a/lib/reencodarr/media/video_queries.ex +++ b/lib/reencodarr/media/video_queries.ex @@ -206,19 +206,16 @@ defmodule Reencodarr.Media.VideoQueries do defp claim_next_video_for_crf_search_in_tx([video_id | rest], opts) do old_snapshot = Reencodarr.Media.fetch_dashboard_video_snapshot_by_id(video_id) - {updated_count, _} = + {updated_count, updated_rows} = from(v in Video, where: v.id == ^video_id and v.state == :analyzed, select: v ) |> Repo.update_all([set: [state: :crf_searching, updated_at: DateTime.utc_now()]], opts) - case updated_count do - 1 -> - case Repo.get(Video, video_id, opts) do - %Video{} = video -> {:claimed, video, old_snapshot} - nil -> :none - end + case {updated_count, updated_rows} do + {1, [video]} -> + {:claimed, video, old_snapshot} _ -> claim_next_video_for_crf_search_in_tx(rest, opts) diff --git a/test/reencodarr/broadway_producers_test.exs b/test/reencodarr/broadway_producers_test.exs index a05b89c1..06388415 100644 --- a/test/reencodarr/broadway_producers_test.exs +++ b/test/reencodarr/broadway_producers_test.exs @@ -150,25 +150,6 @@ defmodule Reencodarr.BroadwayProducersTest do assert new_id == video.id end - test "atomic claim broadcasts video mutation for CRF work" do - Phoenix.PubSub.subscribe(Reencodarr.PubSub, "video_state_transitions") - expect_crf_available() - {:ok, video} = video_fixture(%{state: :analyzed}) - - state = init_state(CrfProducer) - {:noreply, [_claimed], _state2} = CrfProducer.handle_demand(1, state) - - assert_receive {:video_mutated, - %{ - action: :update, - old_video: %{id: old_id, state: :analyzed}, - new_video: %{id: new_id, state: :crf_searching} - }} - - assert old_id == video.id - assert new_id == video.id - end - test "claimed CRF videos are not dispatched again" do expect_crf_available() From 665284547b22ce3e233185f6fed3ba945fd1237b Mon Sep 17 00:00:00 2001 From: Mika Cohen Date: Tue, 7 Jul 2026 13:42:14 -0600 Subject: [PATCH 22/70] Stream worker media chunks --- lib/reencodarr/ab_av1/worker_protocol.ex | 91 +++++++- lib/reencodarr_web/channels/worker_channel.ex | 194 ++++++++++++++++++ .../channels/worker_channel_test.exs | 85 ++++++++ 3 files changed, 369 insertions(+), 1 deletion(-) diff --git a/lib/reencodarr/ab_av1/worker_protocol.ex b/lib/reencodarr/ab_av1/worker_protocol.ex index 8b88edce..5bf261ed 100644 --- a/lib/reencodarr/ab_av1/worker_protocol.ex +++ b/lib/reencodarr/ab_av1/worker_protocol.ex @@ -7,6 +7,7 @@ defmodule Reencodarr.AbAv1.WorkerProtocol do @crf_search_topic "workers:crf_search" @supported_protocol_versions [1] + @default_chunk_size_bytes 1_048_576 defmodule Announcement do @moduledoc false @@ -134,6 +135,9 @@ defmodule Reencodarr.AbAv1.WorkerProtocol do @spec supported_protocol_versions() :: [pos_integer()] def supported_protocol_versions, do: @supported_protocol_versions + @spec chunk_size_bytes() :: pos_integer() + def chunk_size_bytes, do: @default_chunk_size_bytes + @spec valid_topic?(String.t()) :: boolean() def valid_topic?(@crf_search_topic), do: true def valid_topic?(_topic), do: false @@ -264,6 +268,91 @@ defmodule Reencodarr.AbAv1.WorkerProtocol do @spec accepted(pos_integer()) :: map() def accepted(protocol_version), do: %{accepted: true, protocol_version: protocol_version} + @spec transfer_started( + Video.t(), + String.t(), + pos_integer(), + non_neg_integer(), + non_neg_integer() + ) :: + map() + def transfer_started( + %Video{id: video_id, path: path, size: size}, + transfer_id, + chunk_size_bytes, + total_bytes, + total_chunks + ) + when is_integer(video_id) and is_binary(path) do + %{ + status: "transfer_started", + video_id: video_id, + transfer_id: transfer_id, + source_name: Path.basename(path), + size_bytes: size || 0, + chunk_size_bytes: chunk_size_bytes, + total_bytes: total_bytes, + total_chunks: total_chunks + } + end + + @spec transfer_chunk( + Video.t(), + String.t(), + non_neg_integer(), + non_neg_integer(), + non_neg_integer(), + non_neg_integer(), + binary() + ) :: map() + def transfer_chunk( + %Video{id: video_id}, + transfer_id, + chunk_index, + total_chunks, + bytes_sent, + total_bytes, + chunk + ) + when is_integer(video_id) and is_binary(transfer_id) and is_integer(chunk_index) and + is_integer(total_chunks) and is_integer(bytes_sent) and is_integer(total_bytes) and + is_binary(chunk) do + %{ + status: "transfer_chunk", + video_id: video_id, + transfer_id: transfer_id, + chunk_index: chunk_index, + total_chunks: total_chunks, + bytes_sent: bytes_sent, + total_bytes: total_bytes, + crc32: :erlang.crc32(chunk), + data: Base.encode64(chunk) + } + end + + @spec transfer_complete(Video.t(), String.t(), non_neg_integer(), non_neg_integer()) :: map() + def transfer_complete(%Video{id: video_id}, transfer_id, total_bytes, total_chunks) + when is_integer(video_id) and is_binary(transfer_id) do + %{ + status: "transfer_complete", + video_id: video_id, + transfer_id: transfer_id, + total_bytes: total_bytes, + total_chunks: total_chunks + } + end + + @spec transfer_failed(Video.t(), String.t(), String.t()) :: map() + def transfer_failed(%Video{id: video_id}, transfer_id, reason) + when is_integer(video_id) and is_binary(transfer_id) and is_binary(reason) do + %{ + status: "transfer_failed", + video_id: video_id, + transfer_id: transfer_id, + reason: reason + } + end + @spec event_ack(String.t()) :: map() def event_ack(event_name), do: %{accepted: true, event: event_name} @@ -279,7 +368,7 @@ defmodule Reencodarr.AbAv1.WorkerProtocol do video_id: video_id, source_name: Path.basename(path), size_bytes: size || 0, - chunk_size_bytes: 1_048_576, + chunk_size_bytes: chunk_size_bytes(), target_vmaf: target_vmaf } end diff --git a/lib/reencodarr_web/channels/worker_channel.ex b/lib/reencodarr_web/channels/worker_channel.ex index f1059d83..0c93e782 100644 --- a/lib/reencodarr_web/channels/worker_channel.ex +++ b/lib/reencodarr_web/channels/worker_channel.ex @@ -84,6 +84,22 @@ defmodule ReencodarrWeb.WorkerChannel do {:reply, {:error, WorkerProtocol.error(:unsupported_event)}, socket} end + @impl true + def handle_info(:stream_transfer_chunk, %{assigns: %{transfer_io_device: nil}} = socket) do + case open_transfer_stream(socket) do + {:ok, socket} -> + push_transfer_start(socket) + + {:error, socket} -> + {:noreply, socket} + end + end + + def handle_info(:stream_transfer_chunk, %{assigns: %{transfer_io_device: io_device}} = socket) + when not is_nil(io_device) do + read_transfer_chunk(socket, io_device) + end + defp ensure_supported_protocol_version(protocol_version) do if WorkerProtocol.supported_protocol_version?(protocol_version) do :ok @@ -196,6 +212,10 @@ defmodule ReencodarrWeb.WorkerChannel do defp assign_claimed_work(worker_id, socket, video) do target_vmaf = Reencodarr.Rules.vmaf_target(video) + transfer_id = Integer.to_string(video.id) + total_bytes = video.size || 0 + chunk_size_bytes = WorkerProtocol.chunk_size_bytes() + total_chunks = total_chunks(total_bytes, chunk_size_bytes) case WorkerSessions.assign_video(worker_id, video.id) do {:ok, _session} -> @@ -203,6 +223,18 @@ defmodule ReencodarrWeb.WorkerChannel do socket |> assign(:current_video_id, video.id) |> assign(:current_vmaf_target, target_vmaf) + |> assign(:transfer_io_device, nil) + |> assign(:transfer_path, video.path) + |> assign(:transfer_id, transfer_id) + |> assign(:transfer_chunk_size_bytes, chunk_size_bytes) + |> assign(:transfer_total_bytes, total_bytes) + |> assign(:transfer_total_chunks, total_chunks) + |> assign(:transfer_bytes_sent, 0) + |> assign(:transfer_chunk_index, 0) + + if File.exists?(video.path) do + send(self(), :stream_transfer_chunk) + end {:reply, {:ok, WorkerProtocol.work_assigned(video, target_vmaf)}, socket} @@ -366,8 +398,170 @@ defmodule ReencodarrWeb.WorkerChannel do assign(socket, :current_video_id, nil) |> assign(:current_vmaf_target, nil) + |> assign(:transfer_io_device, nil) + |> assign(:transfer_path, nil) + |> assign(:transfer_id, nil) + |> assign(:transfer_chunk_size_bytes, nil) + |> assign(:transfer_total_bytes, nil) + |> assign(:transfer_total_chunks, nil) + |> assign(:transfer_bytes_sent, nil) + |> assign(:transfer_chunk_index, nil) + |> assign(:transfer_started_sent, nil) + end + + defp open_transfer_stream( + %{assigns: %{current_video_id: video_id, transfer_path: path}} = socket + ) do + case File.open(path, [:read, :binary]) do + {:ok, io_device} -> + socket = + socket + |> assign(:transfer_io_device, io_device) + |> assign(:transfer_started_sent, false) + + {:ok, socket} + + {:error, reason} -> + socket = handle_transfer_failure(socket, video_id, reason) + {:error, socket} + end end + defp push_transfer_start(%{assigns: %{current_video_id: video_id}} = socket) do + video = Media.get_video(video_id) + + if is_nil(video) do + {:noreply, handle_transfer_failure(socket, video_id, :enoent)} + else + socket = + socket + |> assign(:transfer_started_sent, true) + + push( + socket, + "transfer_started", + WorkerProtocol.transfer_started( + video, + socket.assigns.transfer_id, + socket.assigns.transfer_chunk_size_bytes, + socket.assigns.transfer_total_bytes, + socket.assigns.transfer_total_chunks + ) + ) + + send(self(), :stream_transfer_chunk) + {:noreply, socket} + end + end + + defp read_transfer_chunk(%{assigns: %{current_video_id: video_id}} = socket, io_device) do + video = Media.get_video(video_id) + + if is_nil(video) do + close_transfer_stream(io_device) + {:noreply, socket} + else + case IO.binread(io_device, socket.assigns.transfer_chunk_size_bytes) do + chunk when is_binary(chunk) -> + chunk_index = socket.assigns.transfer_chunk_index + bytes_sent = socket.assigns.transfer_bytes_sent + byte_size(chunk) + + push( + socket, + "transfer_chunk", + WorkerProtocol.transfer_chunk( + video, + socket.assigns.transfer_id, + chunk_index, + socket.assigns.transfer_total_chunks, + bytes_sent, + socket.assigns.transfer_total_bytes, + chunk + ) + ) + + send(self(), :stream_transfer_chunk) + + {:noreply, + socket + |> assign(:transfer_bytes_sent, bytes_sent) + |> assign(:transfer_chunk_index, chunk_index + 1)} + + :eof -> + close_transfer_stream(io_device) + + push( + socket, + "transfer_complete", + WorkerProtocol.transfer_complete( + video, + socket.assigns.transfer_id, + socket.assigns.transfer_total_bytes, + socket.assigns.transfer_total_chunks + ) + ) + + {:noreply, + socket + |> assign(:transfer_io_device, nil) + |> assign(:transfer_path, nil) + |> assign(:transfer_id, nil) + |> assign(:transfer_chunk_size_bytes, nil) + |> assign(:transfer_total_bytes, nil) + |> assign(:transfer_total_chunks, nil) + |> assign(:transfer_bytes_sent, nil) + |> assign(:transfer_chunk_index, nil) + |> assign(:transfer_started_sent, nil)} + + {:error, reason} -> + close_transfer_stream(io_device) + {:noreply, handle_transfer_failure(socket, video_id, reason)} + end + end + end + + defp handle_transfer_failure(socket, video_id, reason) do + case Media.get_video(video_id) do + nil -> + clear_assigned_video(socket.assigns.worker_id, socket) + + video -> + message = format_file_error(reason) + + _ = + Media.record_video_failure(video, :crf_search, :file_access, + message: "Transfer failed: #{message}", + context: %{ + transfer_id: socket.assigns[:transfer_id], + path: socket.assigns[:transfer_path] + } + ) + + socket = clear_assigned_video(socket.assigns.worker_id, socket) + + _ = + push( + socket, + "transfer_failed", + WorkerProtocol.transfer_failed(video, Integer.to_string(video.id), message) + ) + + socket + end + end + + defp close_transfer_stream(nil), do: :ok + defp close_transfer_stream(io_device), do: File.close(io_device) + + defp format_file_error(reason) do + reason |> :file.format_error() |> List.to_string() + end + + defp total_chunks(0, _chunk_size_bytes), do: 0 + + defp total_chunks(total_bytes, chunk_size_bytes), + do: div(total_bytes + chunk_size_bytes - 1, chunk_size_bytes) + defp vmaf_to_event(%Reencodarr.Media.Vmaf{} = vmaf) do %{ video_id: vmaf.video_id, diff --git a/test/reencodarr_web/channels/worker_channel_test.exs b/test/reencodarr_web/channels/worker_channel_test.exs index e05d20e5..482c2aa3 100644 --- a/test/reencodarr_web/channels/worker_channel_test.exs +++ b/test/reencodarr_web/channels/worker_channel_test.exs @@ -7,6 +7,7 @@ defmodule ReencodarrWeb.WorkerChannelTest do alias Reencodarr.Media alias ReencodarrWeb.WorkerChannel alias ReencodarrWeb.WorkerSocket + import Reencodarr.TestHelpers describe "ab-av1 worker websocket" do setup do @@ -245,6 +246,90 @@ defmodule ReencodarrWeb.WorkerChannelTest do Application.delete_env(:reencodarr, :worker_token) end + test "streams assigned media in ordered chunks with integrity data" do + token = "test-worker-token" + Application.put_env(:reencodarr, :worker_token, token) + + content = :binary.copy("abcd", 262_145) + content_size = byte_size(content) + first_chunk = binary_part(content, 0, 1_048_576) + second_chunk = binary_part(content, 1_048_576, content_size - 1_048_576) + first_crc32 = :erlang.crc32(first_chunk) + second_crc32 = :erlang.crc32(second_chunk) + first_encoded = Base.encode64(first_chunk) + second_encoded = Base.encode64(second_chunk) + + with_temp_file(content, ".mkv", fn path -> + {:ok, video} = + Fixtures.video_fixture(%{ + path: path, + size: content_size, + state: :analyzed + }) + + assert {:ok, socket} = connect(WorkerSocket, %{"token" => token}) + assert {:ok, _join_payload, socket} = subscribe_and_join(socket, "workers:crf_search") + + assert_reply push(socket, "announce", announce_payload(worker_id: "worker-a")), + :ok, + %{accepted: true, protocol_version: 1} + + assert_reply push(socket, "pull_work", %{}), + :ok, + %{ + status: "job_assigned", + video_id: assigned_video_id, + chunk_size_bytes: 1_048_576 + } + + assert assigned_video_id == video.id + + assert_push "transfer_started", %{ + status: "transfer_started", + video_id: ^assigned_video_id, + transfer_id: transfer_id, + size_bytes: content_size, + chunk_size_bytes: 1_048_576, + total_bytes: content_size, + total_chunks: 2 + } + + assert_push "transfer_chunk", %{ + status: "transfer_chunk", + video_id: ^assigned_video_id, + transfer_id: ^transfer_id, + chunk_index: 0, + total_chunks: 2, + bytes_sent: 1_048_576, + total_bytes: ^content_size, + crc32: ^first_crc32, + data: ^first_encoded + } + + assert_push "transfer_chunk", %{ + status: "transfer_chunk", + video_id: ^assigned_video_id, + transfer_id: ^transfer_id, + chunk_index: 1, + total_chunks: 2, + bytes_sent: ^content_size, + total_bytes: ^content_size, + crc32: ^second_crc32, + data: ^second_encoded + } + + assert_push "transfer_complete", %{ + status: "transfer_complete", + video_id: ^assigned_video_id, + transfer_id: ^transfer_id, + total_bytes: ^content_size, + total_chunks: 2 + } + end) + after + Application.delete_env(:reencodarr, :worker_token) + end + test "submits structured CRF results and completes the job" do token = "test-worker-token" Application.put_env(:reencodarr, :worker_token, token) From 4920717ea8b5ecd6ab799a383762938841c4d46d Mon Sep 17 00:00:00 2001 From: Mika Cohen Date: Tue, 7 Jul 2026 13:55:42 -0600 Subject: [PATCH 23/70] Tighten worker websocket rollout --- config/config.exs | 8 +- config/runtime.exs | 23 ++++ docs/distributed_worker_protocol.md | 121 ++++++++++++++++++ lib/reencodarr/ab_av1/worker_config.ex | 37 ++++++ lib/reencodarr/ab_av1/worker_protocol.ex | 8 +- lib/reencodarr_web/channels/worker_socket.ex | 18 ++- .../ab_av1/worker_protocol_test.exs | 15 +++ .../channels/worker_channel_test.exs | 24 ++++ 8 files changed, 239 insertions(+), 15 deletions(-) create mode 100644 docs/distributed_worker_protocol.md create mode 100644 lib/reencodarr/ab_av1/worker_config.ex diff --git a/config/config.exs b/config/config.exs index ab9602bb..cb291da6 100644 --- a/config/config.exs +++ b/config/config.exs @@ -12,7 +12,13 @@ config :reencodarr, generators: [timestamp_type: :utc_datetime], env: config_env(), dashboard_stats_query_timeout_ms: 500, - dashboard_queue_query_timeout_ms: 5_000 + dashboard_queue_query_timeout_ms: 5_000, + distributed_worker_enabled: true, + worker_chunk_size_bytes: 1_048_576, + worker_transfer_window: 1, + worker_retry_limit: 3, + worker_transfer_timeout_ms: 60_000, + worker_max_concurrent_transfers: 1 # Shared database configuration - SQLite performance tuning config :reencodarr, Reencodarr.Repo, diff --git a/config/runtime.exs b/config/runtime.exs index 85581309..59b0766c 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -20,6 +20,20 @@ if System.get_env("PHX_SERVER") do config :reencodarr, ReencodarrWeb.Endpoint, server: true end +parse_bool_env = fn + "false" -> false + "0" -> false + "no" -> false + _ -> true +end + +parse_int_env = fn name, default -> + case System.get_env(name) do + nil -> default + value -> String.to_integer(value) + end +end + worker_token = System.get_env("REENCODARR_WORKER_TOKEN") |> case do @@ -41,6 +55,15 @@ if worker_token do config :reencodarr, :worker_token, worker_token end +config :reencodarr, + distributed_worker_enabled: + parse_bool_env.(System.get_env("REENCODARR_DISTRIBUTED_WORKERS", "true")), + worker_chunk_size_bytes: parse_int_env.("REENCODARR_WORKER_CHUNK_SIZE_BYTES", 1_048_576), + worker_transfer_window: parse_int_env.("REENCODARR_WORKER_TRANSFER_WINDOW", 1), + worker_retry_limit: parse_int_env.("REENCODARR_WORKER_RETRY_LIMIT", 3), + worker_transfer_timeout_ms: parse_int_env.("REENCODARR_WORKER_TRANSFER_TIMEOUT_MS", 60_000), + worker_max_concurrent_transfers: parse_int_env.("REENCODARR_WORKER_MAX_CONCURRENT_TRANSFERS", 1) + if config_env() == :prod do # Parse CHECK_ORIGIN env var: "false" disables, otherwise comma-separated origin list parse_check_origin = fn diff --git a/docs/distributed_worker_protocol.md b/docs/distributed_worker_protocol.md new file mode 100644 index 00000000..ecfbb842 --- /dev/null +++ b/docs/distributed_worker_protocol.md @@ -0,0 +1,121 @@ +# Distributed CRF-search Websocket Protocol + +This document describes the current Reencodarr websocket contract for distributed +`ab-av1` CRF-search workers. + +## Versioning + +- Protocol version: `1` +- Transport: Phoenix websocket channel `workers:crf_search` +- Worker authentication: token-based socket connect, then per-worker channel join + +## Connection Flow + +1. Client connects to the websocket with the shared worker token. +2. Client joins `workers:crf_search`. +3. Client sends `announce`. +4. Server registers the worker session and returns `accepted: true`. +5. Client requests work with `pull_work` or `request_work`. +6. Server assigns work or returns `no_work`. + +## Announce + +Client payload: + +- `worker_id` +- `protocol_version` +- `version` +- `capabilities` + +Server accepts protocol version `1` and rejects unsupported versions. + +## Job Assignment + +Server reply: + +- `status: "job_assigned"` +- `job_id` +- `video_id` +- `source_name` +- `size_bytes` +- `chunk_size_bytes` +- `target_vmaf` + +The current chunk size is `1_048_576` bytes. + +## Media Transfer + +After assignment, the server streams the source file on the same websocket. + +### `transfer_started` + +Fields: + +- `status: "transfer_started"` +- `video_id` +- `transfer_id` +- `source_name` +- `size_bytes` +- `chunk_size_bytes` +- `total_bytes` +- `total_chunks` + +### `transfer_chunk` + +Fields: + +- `status: "transfer_chunk"` +- `video_id` +- `transfer_id` +- `chunk_index` +- `total_chunks` +- `bytes_sent` +- `total_bytes` +- `crc32` +- `data` + +Notes: + +- `chunk_index` starts at `0` and increments monotonically. +- `crc32` is computed from the raw chunk bytes. +- `data` is base64-encoded chunk data. + +### `transfer_complete` + +Fields: + +- `status: "transfer_complete"` +- `video_id` +- `transfer_id` +- `total_bytes` +- `total_chunks` + +### `transfer_failed` + +Fields: + +- `status: "transfer_failed"` +- `video_id` +- `transfer_id` +- `reason` + +Transfer failures are recorded as file-access failures on the server side. + +## Worker Events + +The channel also accepts these client-to-server events: + +- `heartbeat` +- `transfer_progress` +- `crf_search_progress` +- `crf_search_result` +- `crf_search_completed` +- `video_failed` + +## Terminal State Rules + +- `crf_search_completed` with `ok` marks the video searched and clears the active assignment. +- `crf_search_completed` with `cancelled` or `shutdown` returns the video to `analyzed`. +- `video_failed` records a structured failure and clears the active assignment. +- Disconnecting a worker with active CRF-search work requeues the video. + diff --git a/lib/reencodarr/ab_av1/worker_config.ex b/lib/reencodarr/ab_av1/worker_config.ex new file mode 100644 index 00000000..a6cc7dcf --- /dev/null +++ b/lib/reencodarr/ab_av1/worker_config.ex @@ -0,0 +1,37 @@ +defmodule Reencodarr.AbAv1.WorkerConfig do + @moduledoc """ + Runtime configuration for distributed ab-av1 workers. + """ + + @default_chunk_size_bytes 1_048_576 + + @spec enabled?() :: boolean() + def enabled? do + Application.get_env(:reencodarr, :distributed_worker_enabled, true) + end + + @spec chunk_size_bytes() :: pos_integer() + def chunk_size_bytes do + Application.get_env(:reencodarr, :worker_chunk_size_bytes, @default_chunk_size_bytes) + end + + @spec transfer_window() :: pos_integer() + def transfer_window do + Application.get_env(:reencodarr, :worker_transfer_window, 1) + end + + @spec retry_limit() :: non_neg_integer() + def retry_limit do + Application.get_env(:reencodarr, :worker_retry_limit, 3) + end + + @spec transfer_timeout_ms() :: pos_integer() + def transfer_timeout_ms do + Application.get_env(:reencodarr, :worker_transfer_timeout_ms, 60_000) + end + + @spec max_concurrent_transfers() :: pos_integer() + def max_concurrent_transfers do + Application.get_env(:reencodarr, :worker_max_concurrent_transfers, 1) + end +end diff --git a/lib/reencodarr/ab_av1/worker_protocol.ex b/lib/reencodarr/ab_av1/worker_protocol.ex index 5bf261ed..a933e65a 100644 --- a/lib/reencodarr/ab_av1/worker_protocol.ex +++ b/lib/reencodarr/ab_av1/worker_protocol.ex @@ -3,11 +3,11 @@ defmodule Reencodarr.AbAv1.WorkerProtocol do Server-side helpers for the ab-av1 worker websocket protocol. """ + alias Reencodarr.AbAv1.WorkerConfig alias Reencodarr.Media.Video @crf_search_topic "workers:crf_search" @supported_protocol_versions [1] - @default_chunk_size_bytes 1_048_576 defmodule Announcement do @moduledoc false @@ -110,7 +110,7 @@ defmodule Reencodarr.AbAv1.WorkerProtocol do @type t :: %__MODULE__{ video_id: pos_integer(), - result: :ok | :cancelled | :shutdown | {:error, term()}, + result: :ok | :cancelled | :shutdown | :failed | {:error, term()}, chosen_crf: number() | nil } end @@ -127,7 +127,7 @@ defmodule Reencodarr.AbAv1.WorkerProtocol do optional(:target) => integer() | nil, optional(:chosen) => boolean() } - @type completion_result :: :ok | :cancelled | :shutdown | {:error, term()} + @type completion_result :: :ok | :cancelled | :shutdown | :failed | {:error, term()} @spec crf_search_topic() :: String.t() def crf_search_topic, do: @crf_search_topic @@ -136,7 +136,7 @@ defmodule Reencodarr.AbAv1.WorkerProtocol do def supported_protocol_versions, do: @supported_protocol_versions @spec chunk_size_bytes() :: pos_integer() - def chunk_size_bytes, do: @default_chunk_size_bytes + def chunk_size_bytes, do: WorkerConfig.chunk_size_bytes() @spec valid_topic?(String.t()) :: boolean() def valid_topic?(@crf_search_topic), do: true diff --git a/lib/reencodarr_web/channels/worker_socket.ex b/lib/reencodarr_web/channels/worker_socket.ex index 839c4e08..9ae2d72e 100644 --- a/lib/reencodarr_web/channels/worker_socket.ex +++ b/lib/reencodarr_web/channels/worker_socket.ex @@ -5,22 +5,20 @@ defmodule ReencodarrWeb.WorkerSocket do use Phoenix.Socket + alias Reencodarr.AbAv1.WorkerConfig alias Reencodarr.AbAv1.WorkerProtocol channel WorkerProtocol.crf_search_topic(), ReencodarrWeb.WorkerChannel @impl true def connect(%{"token" => token}, socket, _connect_info) when is_binary(token) do - case Application.fetch_env(:reencodarr, :worker_token) do - {:ok, configured_token} when is_binary(configured_token) -> - if Plug.Crypto.secure_compare(configured_token, token) do - {:ok, assign(socket, :worker_id, worker_id())} - else - :error - end - - _ -> - :error + with true <- WorkerConfig.enabled?(), + {:ok, configured_token} when is_binary(configured_token) <- + Application.fetch_env(:reencodarr, :worker_token), + true <- Plug.Crypto.secure_compare(configured_token, token) do + {:ok, assign(socket, :worker_id, worker_id())} + else + _ -> :error end end diff --git a/test/reencodarr/ab_av1/worker_protocol_test.exs b/test/reencodarr/ab_av1/worker_protocol_test.exs index 8227fe03..773de72b 100644 --- a/test/reencodarr/ab_av1/worker_protocol_test.exs +++ b/test/reencodarr/ab_av1/worker_protocol_test.exs @@ -157,4 +157,19 @@ defmodule Reencodarr.AbAv1.WorkerProtocolTest do target_vmaf: 96.5 } end + + test "reads chunk size from configuration" do + previous = Application.get_env(:reencodarr, :worker_chunk_size_bytes) + Application.put_env(:reencodarr, :worker_chunk_size_bytes, 2_097_152) + + try do + assert WorkerProtocol.chunk_size_bytes() == 2_097_152 + after + if is_nil(previous) do + Application.delete_env(:reencodarr, :worker_chunk_size_bytes) + else + Application.put_env(:reencodarr, :worker_chunk_size_bytes, previous) + end + end + end end diff --git a/test/reencodarr_web/channels/worker_channel_test.exs b/test/reencodarr_web/channels/worker_channel_test.exs index 482c2aa3..f4ceee02 100644 --- a/test/reencodarr_web/channels/worker_channel_test.exs +++ b/test/reencodarr_web/channels/worker_channel_test.exs @@ -502,6 +502,30 @@ defmodule ReencodarrWeb.WorkerChannelTest do Application.delete_env(:reencodarr, :worker_token) end + test "rejects worker connections when distributed workers are disabled" do + previous_enabled = Application.get_env(:reencodarr, :distributed_worker_enabled) + previous_token = Application.get_env(:reencodarr, :worker_token) + + on_exit(fn -> + if is_nil(previous_enabled) do + Application.delete_env(:reencodarr, :distributed_worker_enabled) + else + Application.put_env(:reencodarr, :distributed_worker_enabled, previous_enabled) + end + + if is_nil(previous_token) do + Application.delete_env(:reencodarr, :worker_token) + else + Application.put_env(:reencodarr, :worker_token, previous_token) + end + end) + + Application.put_env(:reencodarr, :distributed_worker_enabled, false) + Application.put_env(:reencodarr, :worker_token, "test-worker-token") + + assert :error = connect(WorkerSocket, %{"token" => "test-worker-token"}) + end + test "rejects invalid topics without crashing" do assert {:error, %{reason: "unauthorized"}} = WorkerChannel.join("workers:other", %{}, %{assigns: %{worker_id: "worker-1"}}) From 13db1dbda2fd11a528f7f196c99d79ff418a7de4 Mon Sep 17 00:00:00 2001 From: Mika Cohen Date: Tue, 7 Jul 2026 14:05:48 -0600 Subject: [PATCH 24/70] Address distributed worker review feedback --- config/config.exs | 2 +- config/runtime.exs | 2 +- lib/reencodarr/ab_av1/worker_config.ex | 2 +- lib/reencodarr/ab_av1/worker_protocol.ex | 8 ++++- lib/reencodarr/ab_av1/worker_sessions.ex | 7 +++- lib/reencodarr_web/live/dashboard_live.ex | 33 +++++++++++-------- .../ab_av1/worker_protocol_test.exs | 24 ++++++++++++++ .../ab_av1/worker_sessions_test.exs | 4 ++- .../channels/worker_channel_test.exs | 11 +++++++ 9 files changed, 74 insertions(+), 19 deletions(-) diff --git a/config/config.exs b/config/config.exs index cb291da6..dea90bd7 100644 --- a/config/config.exs +++ b/config/config.exs @@ -13,7 +13,7 @@ config :reencodarr, env: config_env(), dashboard_stats_query_timeout_ms: 500, dashboard_queue_query_timeout_ms: 5_000, - distributed_worker_enabled: true, + distributed_worker_enabled: false, worker_chunk_size_bytes: 1_048_576, worker_transfer_window: 1, worker_retry_limit: 3, diff --git a/config/runtime.exs b/config/runtime.exs index 59b0766c..4eafd4ee 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -57,7 +57,7 @@ end config :reencodarr, distributed_worker_enabled: - parse_bool_env.(System.get_env("REENCODARR_DISTRIBUTED_WORKERS", "true")), + parse_bool_env.(System.get_env("REENCODARR_DISTRIBUTED_WORKERS", "false")), worker_chunk_size_bytes: parse_int_env.("REENCODARR_WORKER_CHUNK_SIZE_BYTES", 1_048_576), worker_transfer_window: parse_int_env.("REENCODARR_WORKER_TRANSFER_WINDOW", 1), worker_retry_limit: parse_int_env.("REENCODARR_WORKER_RETRY_LIMIT", 3), diff --git a/lib/reencodarr/ab_av1/worker_config.ex b/lib/reencodarr/ab_av1/worker_config.ex index a6cc7dcf..08e53017 100644 --- a/lib/reencodarr/ab_av1/worker_config.ex +++ b/lib/reencodarr/ab_av1/worker_config.ex @@ -7,7 +7,7 @@ defmodule Reencodarr.AbAv1.WorkerConfig do @spec enabled?() :: boolean() def enabled? do - Application.get_env(:reencodarr, :distributed_worker_enabled, true) + Application.get_env(:reencodarr, :distributed_worker_enabled, false) end @spec chunk_size_bytes() :: pos_integer() diff --git a/lib/reencodarr/ab_av1/worker_protocol.ex b/lib/reencodarr/ab_av1/worker_protocol.ex index a933e65a..bfa89c78 100644 --- a/lib/reencodarr/ab_av1/worker_protocol.ex +++ b/lib/reencodarr/ab_av1/worker_protocol.ex @@ -660,6 +660,12 @@ defmodule Reencodarr.AbAv1.WorkerProtocol do end defp fetch_any(payload, keys) do - Enum.find_value(keys, fn key -> Map.get(payload, key) end) + Enum.find_value(keys, fn key -> + if Map.has_key?(payload, key), do: {:ok, Map.fetch!(payload, key)} + end) + |> case do + {:ok, value} -> value + nil -> nil + end end end diff --git a/lib/reencodarr/ab_av1/worker_sessions.ex b/lib/reencodarr/ab_av1/worker_sessions.ex index 274d3806..5504b5d0 100644 --- a/lib/reencodarr/ab_av1/worker_sessions.ex +++ b/lib/reencodarr/ab_av1/worker_sessions.ex @@ -144,7 +144,12 @@ defmodule Reencodarr.AbAv1.WorkerSessions do @impl GenServer def handle_info(:expire_stale, state) do - _ = expire_stale_sessions(timeout_seconds()) + {expired_sessions, _} = expire_stale_sessions(timeout_seconds()) + + if expired_sessions != [] do + broadcast_sessions() + end + schedule_expire_stale() {:noreply, state} end diff --git a/lib/reencodarr_web/live/dashboard_live.ex b/lib/reencodarr_web/live/dashboard_live.ex index e9313bdf..898f72d8 100644 --- a/lib/reencodarr_web/live/dashboard_live.ex +++ b/lib/reencodarr_web/live/dashboard_live.ex @@ -51,7 +51,7 @@ defmodule ReencodarrWeb.DashboardLive do sync_progress: 0, service_type: nil, page_title: nil, - worker_token: Application.get_env(:reencodarr, :worker_token), + worker_token_state: worker_token_state(), worker_socket_url: worker_socket_url(), # New dashboard stats stats: Reencodarr.Media.get_default_stats(), @@ -988,7 +988,7 @@ defmodule ReencodarrWeb.DashboardLive do attr :syncing, :boolean, required: true attr :sync_progress, :integer, required: true attr :service_type, :atom, required: true - attr :worker_token, :string, default: nil + attr :worker_token_state, :any, required: true attr :worker_socket_url, :string, required: true defp sync_controls(assigns) do @@ -1053,10 +1053,11 @@ defmodule ReencodarrWeb.DashboardLive do
Token fingerprint
- <%= if @worker_token do %> - {worker_token_fingerprint(@worker_token)} - <% else %> - not configured + <%= case @worker_token_state do %> + <% {:ok, fingerprint} -> %> + {fingerprint} + <% :error -> %> + not configured <% end %>
@@ -1091,13 +1092,19 @@ defmodule ReencodarrWeb.DashboardLive do defp format_number(_), do: "—" - defp worker_token_fingerprint(token) when is_binary(token) do - digest = - :crypto.hash(:sha256, token) - |> Base.encode16(case: :lower) - |> String.slice(0, 12) + defp worker_token_state do + case Application.get_env(:reencodarr, :worker_token) do + token when is_binary(token) -> + digest = + :crypto.hash(:sha256, token) + |> Base.encode16(case: :lower) + |> String.slice(0, 12) + + {:ok, "sha256:#{digest}"} - "sha256:#{digest}" + _ -> + :error + end end defp format_completed(stats) do @@ -1276,7 +1283,7 @@ defmodule ReencodarrWeb.DashboardLive do syncing={@syncing} sync_progress={@sync_progress} service_type={@service_type} - worker_token={@worker_token} + worker_token_state={@worker_token_state} worker_socket_url={@worker_socket_url} />
diff --git a/test/reencodarr/ab_av1/worker_protocol_test.exs b/test/reencodarr/ab_av1/worker_protocol_test.exs index 773de72b..62547c98 100644 --- a/test/reencodarr/ab_av1/worker_protocol_test.exs +++ b/test/reencodarr/ab_av1/worker_protocol_test.exs @@ -110,6 +110,30 @@ defmodule Reencodarr.AbAv1.WorkerProtocolTest do "result" => "ok", "chosen_crf" => 28 }) + + assert {:ok, + %CrfSearchResult{ + results: [%{chosen: false}] + }} = + WorkerProtocol.parse_crf_search_result(%{ + "video_id" => 123, + "crf" => 28, + "score" => 95.4, + "percent" => 94, + "chosen" => false + }) + + assert {:ok, + %FailureReport{ + retriable: false + }} = + WorkerProtocol.parse_failure_report(%{ + "video_id" => 123, + "stage" => "crf_search", + "category" => "timeout", + "message" => "timed out", + "retriable" => false + }) end test "parses typed failure reports" do diff --git a/test/reencodarr/ab_av1/worker_sessions_test.exs b/test/reencodarr/ab_av1/worker_sessions_test.exs index 950c0ad0..04534856 100644 --- a/test/reencodarr/ab_av1/worker_sessions_test.exs +++ b/test/reencodarr/ab_av1/worker_sessions_test.exs @@ -2,6 +2,7 @@ defmodule Reencodarr.AbAv1.WorkerSessionsTest do use Reencodarr.DataCase, async: false alias Reencodarr.AbAv1.WorkerSessions + alias Reencodarr.Dashboard.Events alias Reencodarr.Diagnostics setup do @@ -40,10 +41,11 @@ defmodule Reencodarr.AbAv1.WorkerSessionsTest do Application.put_env(:reencodarr, :worker_session_timeout_seconds, previous_timeout) end) + Phoenix.PubSub.subscribe(Reencodarr.PubSub, Events.channel()) assert {:ok, _session} = WorkerSessions.register(worker_session_attrs()) send(Process.whereis(WorkerSessions), :expire_stale) - :timer.sleep(25) + assert_receive {:worker_sessions_updated, %{sessions: []}} assert WorkerSessions.list() == [] end diff --git a/test/reencodarr_web/channels/worker_channel_test.exs b/test/reencodarr_web/channels/worker_channel_test.exs index f4ceee02..cfcc2044 100644 --- a/test/reencodarr_web/channels/worker_channel_test.exs +++ b/test/reencodarr_web/channels/worker_channel_test.exs @@ -11,7 +11,18 @@ defmodule ReencodarrWeb.WorkerChannelTest do describe "ab-av1 worker websocket" do setup do + previous_enabled = Application.get_env(:reencodarr, :distributed_worker_enabled) + Application.put_env(:reencodarr, :distributed_worker_enabled, true) WorkerSessions.reset() + + on_exit(fn -> + if is_nil(previous_enabled) do + Application.delete_env(:reencodarr, :distributed_worker_enabled) + else + Application.put_env(:reencodarr, :distributed_worker_enabled, previous_enabled) + end + end) + :ok end From 898ff54691f4fbbb0814548d50e2992d2d6af503 Mon Sep 17 00:00:00 2001 From: Mika Cohen Date: Tue, 7 Jul 2026 14:18:50 -0600 Subject: [PATCH 25/70] Show worker session state --- lib/reencodarr/diagnostics.ex | 12 +++++++++++- lib/reencodarr_web/live/workers_live.ex | 11 +++++++++-- test/reencodarr/ab_av1/worker_sessions_test.exs | 4 ++++ test/reencodarr_web/live/workers_live_test.exs | 10 +++++++--- 4 files changed, 31 insertions(+), 6 deletions(-) diff --git a/lib/reencodarr/diagnostics.ex b/lib/reencodarr/diagnostics.ex index fd3c1a36..2c147aa3 100644 --- a/lib/reencodarr/diagnostics.ex +++ b/lib/reencodarr/diagnostics.ex @@ -670,11 +670,21 @@ defmodule Reencodarr.Diagnostics do defp format_worker_sessions(worker_sessions) when is_list(worker_sessions) do Enum.map_join(worker_sessions, "\n", fn session -> active_video_id = session.active_video_id || "none" + active_video_state = worker_session_state(session.active_video_id) - " #{session.client_worker_id} protocol=#{session.protocol_version} version=#{session.version} video=#{active_video_id} capabilities=#{inspect(session.capabilities)} last_seen=#{DateTime.to_iso8601(session.last_seen_at)}" + " #{session.client_worker_id} protocol=#{session.protocol_version} version=#{session.version} video=#{active_video_id} state=#{active_video_state} capabilities=#{inspect(session.capabilities)} last_seen=#{DateTime.to_iso8601(session.last_seen_at)}" end) end + defp worker_session_state(nil), do: "idle" + + defp worker_session_state(video_id) do + case Reencodarr.Media.get_video(video_id) do + %Video{state: state} -> Atom.to_string(state) + nil -> "missing" + end + end + defp video_active_in_state?({:error, _}, _video_id), do: false defp video_active_in_state?(state, video_id) when is_map(state) do diff --git a/lib/reencodarr_web/live/workers_live.ex b/lib/reencodarr_web/live/workers_live.ex index 42fd5b10..5a837e1a 100644 --- a/lib/reencodarr_web/live/workers_live.ex +++ b/lib/reencodarr_web/live/workers_live.ex @@ -7,6 +7,7 @@ defmodule ReencodarrWeb.WorkersLive do alias Reencodarr.AbAv1.WorkerSessions alias Reencodarr.Dashboard.Events + alias Reencodarr.Media @refresh_interval 5_000 @@ -67,7 +68,7 @@ defmodule ReencodarrWeb.WorkersLive do Worker - Status + State Video Protocol Version @@ -123,7 +124,13 @@ defmodule ReencodarrWeb.WorkersLive do end defp worker_status(%{active_video_id: nil}), do: "Idle" - defp worker_status(%{active_video_id: _video_id}), do: "Working" + + defp worker_status(%{active_video_id: video_id}) do + case Media.get_video(video_id) do + %Media.Video{state: state} -> Atom.to_string(state) + nil -> "missing" + end + end defp worker_video(%{active_video_id: nil}), do: "none" defp worker_video(%{active_video_id: video_id}), do: "video ##{video_id}" diff --git a/test/reencodarr/ab_av1/worker_sessions_test.exs b/test/reencodarr/ab_av1/worker_sessions_test.exs index 04534856..889d4f7f 100644 --- a/test/reencodarr/ab_av1/worker_sessions_test.exs +++ b/test/reencodarr/ab_av1/worker_sessions_test.exs @@ -4,6 +4,7 @@ defmodule Reencodarr.AbAv1.WorkerSessionsTest do alias Reencodarr.AbAv1.WorkerSessions alias Reencodarr.Dashboard.Events alias Reencodarr.Diagnostics + alias Reencodarr.Fixtures setup do WorkerSessions.reset() @@ -52,6 +53,8 @@ defmodule Reencodarr.AbAv1.WorkerSessionsTest do test "includes worker sessions in diagnostics output" do assert {:ok, _session} = WorkerSessions.register(worker_session_attrs()) + {:ok, video} = Fixtures.video_fixture(%{state: :analyzed}) + assert {:ok, _session} = WorkerSessions.assign_video("worker-server-1", video.id) output = Diagnostics.processes() @@ -59,6 +62,7 @@ defmodule Reencodarr.AbAv1.WorkerSessionsTest do assert output =~ "worker-client-1" assert output =~ "protocol=1" assert output =~ "version=0.10.0" + assert output =~ "state=analyzed" end test "tracks an assigned video on the session" do diff --git a/test/reencodarr_web/live/workers_live_test.exs b/test/reencodarr_web/live/workers_live_test.exs index 3f4dbb11..6efa66ec 100644 --- a/test/reencodarr_web/live/workers_live_test.exs +++ b/test/reencodarr_web/live/workers_live_test.exs @@ -4,6 +4,7 @@ defmodule ReencodarrWeb.WorkersLiveTest do import Phoenix.LiveViewTest alias Reencodarr.AbAv1.WorkerSessions + alias Reencodarr.Fixtures setup do WorkerSessions.reset() @@ -28,14 +29,17 @@ defmodule ReencodarrWeb.WorkersLiveTest do capabilities: %{"crf_search" => true} }) + {:ok, video} = Fixtures.video_fixture(%{state: :analyzed}) + assert {:ok, view, html} = live(conn, ~p"/workers") assert html =~ "worker-client-1" assert html =~ "Idle" - assert {:ok, _session} = WorkerSessions.assign_video("worker-server-1", 123) + assert {:ok, _session} = WorkerSessions.assign_video("worker-server-1", video.id) + send(view.pid, {:worker_sessions_updated, %{sessions: WorkerSessions.list()}}) html = render(view) - assert html =~ "Working" - assert html =~ "video #123" + assert html =~ "analyzed" + assert html =~ "video ##{video.id}" end end From e0d1ad6368143672f1de399e02d8ac9148f4cc60 Mon Sep 17 00:00:00 2001 From: Mika Cohen Date: Tue, 7 Jul 2026 14:31:51 -0600 Subject: [PATCH 26/70] Add distributed worker controls --- lib/reencodarr/ab_av1/worker_config.ex | 12 +++ lib/reencodarr/ab_av1/worker_sessions.ex | 79 ++++++++++++++++++- lib/reencodarr/diagnostics.ex | 14 +++- lib/reencodarr_web/channels/worker_channel.ex | 2 + test/reencodarr/ab_av1/worker_config_test.exs | 23 ++++++ .../ab_av1/worker_sessions_test.exs | 28 +++++++ .../channels/worker_channel_test.exs | 4 + 7 files changed, 159 insertions(+), 3 deletions(-) create mode 100644 test/reencodarr/ab_av1/worker_config_test.exs diff --git a/lib/reencodarr/ab_av1/worker_config.ex b/lib/reencodarr/ab_av1/worker_config.ex index 08e53017..62485d44 100644 --- a/lib/reencodarr/ab_av1/worker_config.ex +++ b/lib/reencodarr/ab_av1/worker_config.ex @@ -10,6 +10,18 @@ defmodule Reencodarr.AbAv1.WorkerConfig do Application.get_env(:reencodarr, :distributed_worker_enabled, false) end + @spec enable() :: :ok + def enable do + Application.put_env(:reencodarr, :distributed_worker_enabled, true) + :ok + end + + @spec disable() :: :ok + def disable do + Application.put_env(:reencodarr, :distributed_worker_enabled, false) + :ok + end + @spec chunk_size_bytes() :: pos_integer() def chunk_size_bytes do Application.get_env(:reencodarr, :worker_chunk_size_bytes, @default_chunk_size_bytes) diff --git a/lib/reencodarr/ab_av1/worker_sessions.ex b/lib/reencodarr/ab_av1/worker_sessions.ex index 5504b5d0..f9d50627 100644 --- a/lib/reencodarr/ab_av1/worker_sessions.ex +++ b/lib/reencodarr/ab_av1/worker_sessions.ex @@ -17,6 +17,8 @@ defmodule Reencodarr.AbAv1.WorkerSessions do protocol_version: pos_integer(), capabilities: map(), active_video_id: integer() | nil, + transfer_progress: map() | nil, + crf_search_progress: map() | nil, connected_at: DateTime.t(), last_seen_at: DateTime.t() } @@ -45,6 +47,22 @@ defmodule Reencodarr.AbAv1.WorkerSessions do GenServer.call(__MODULE__, {:clear_video, server_worker_id}) end + def set_transfer_progress(server_worker_id, progress) when is_map(progress) do + GenServer.call(__MODULE__, {:set_transfer_progress, server_worker_id, progress}) + end + + def set_crf_search_progress(server_worker_id, progress) when is_map(progress) do + GenServer.call(__MODULE__, {:set_crf_search_progress, server_worker_id, progress}) + end + + def cancel(server_worker_id) do + GenServer.call(__MODULE__, {:cancel, server_worker_id}) + end + + def drain do + GenServer.call(__MODULE__, :drain) + end + def get(server_worker_id) do GenServer.call(__MODULE__, {:get, server_worker_id}) end @@ -107,16 +125,41 @@ defmodule Reencodarr.AbAv1.WorkerSessions do def handle_call({:assign_video, server_worker_id, video_id}, _from, state) do update_session_reply(server_worker_id, state, fn session -> - %{session | active_video_id: video_id} + %{session | active_video_id: video_id, transfer_progress: nil, crf_search_progress: nil} end) end def handle_call({:clear_video, server_worker_id}, _from, state) do update_session_reply(server_worker_id, state, fn session -> - %{session | active_video_id: nil} + %{session | active_video_id: nil, transfer_progress: nil, crf_search_progress: nil} + end) + end + + def handle_call({:set_transfer_progress, server_worker_id, progress}, _from, state) do + update_session_reply(server_worker_id, state, fn session -> + %{session | transfer_progress: progress} + end) + end + + def handle_call({:set_crf_search_progress, server_worker_id, progress}, _from, state) do + update_session_reply(server_worker_id, state, fn session -> + %{session | crf_search_progress: progress} end) end + def handle_call({:cancel, server_worker_id}, _from, state) do + case lookup_session(server_worker_id) do + {:ok, session} -> + requeue_active_video(session) + :ok = drop_session(server_worker_id) + broadcast_sessions() + {:reply, {:ok, session}, state} + + :error -> + {:reply, {:error, :unknown_worker_session}, state} + end + end + def handle_call({:get, server_worker_id}, _from, state) do {:reply, session_or_nil(lookup_session(server_worker_id)), state} end @@ -142,6 +185,23 @@ defmodule Reencodarr.AbAv1.WorkerSessions do {:reply, :ok, state} end + def handle_call(:drain, _from, state) do + drained_sessions = + list_sessions() + |> Enum.filter(&(&1.active_video_id != nil)) + |> Enum.map(fn session -> + requeue_active_video(session) + :ok = drop_session(session.server_worker_id) + session + end) + + if drained_sessions != [] do + broadcast_sessions() + end + + {:reply, {:ok, drained_sessions}, state} + end + @impl GenServer def handle_info(:expire_stale, state) do {expired_sessions, _} = expire_stale_sessions(timeout_seconds()) @@ -169,6 +229,8 @@ defmodule Reencodarr.AbAv1.WorkerSessions do protocol_version: protocol_version, capabilities: capabilities, active_video_id: nil, + transfer_progress: nil, + crf_search_progress: nil, connected_at: now, last_seen_at: now } @@ -242,6 +304,19 @@ defmodule Reencodarr.AbAv1.WorkerSessions do Events.broadcast_event(:worker_sessions_updated, %{sessions: list_sessions()}) end + defp requeue_active_video(%{active_video_id: nil}), do: :ok + + defp requeue_active_video(%{active_video_id: video_id}) do + case Reencodarr.Media.get_video(video_id) do + %Reencodarr.Media.Video{} = video -> + _ = Reencodarr.Media.mark_as_analyzed(video) + :ok + + nil -> + :ok + end + end + defp put_session(session) do :ok = drop_session(session.server_worker_id) true = :ets.insert(@by_server_table, {session.server_worker_id, session}) diff --git a/lib/reencodarr/diagnostics.ex b/lib/reencodarr/diagnostics.ex index 2c147aa3..c5c06310 100644 --- a/lib/reencodarr/diagnostics.ex +++ b/lib/reencodarr/diagnostics.ex @@ -671,8 +671,10 @@ defmodule Reencodarr.Diagnostics do Enum.map_join(worker_sessions, "\n", fn session -> active_video_id = session.active_video_id || "none" active_video_state = worker_session_state(session.active_video_id) + transfer_progress = format_worker_progress(session.transfer_progress, "transfer") + crf_search_progress = format_worker_progress(session.crf_search_progress, "crf") - " #{session.client_worker_id} protocol=#{session.protocol_version} version=#{session.version} video=#{active_video_id} state=#{active_video_state} capabilities=#{inspect(session.capabilities)} last_seen=#{DateTime.to_iso8601(session.last_seen_at)}" + " #{session.client_worker_id} protocol=#{session.protocol_version} version=#{session.version} video=#{active_video_id} state=#{active_video_state} transfer=#{transfer_progress} crf=#{crf_search_progress} capabilities=#{inspect(session.capabilities)} last_seen=#{DateTime.to_iso8601(session.last_seen_at)}" end) end @@ -685,6 +687,16 @@ defmodule Reencodarr.Diagnostics do end end + defp format_worker_progress(nil, _label), do: "none" + + defp format_worker_progress(progress, label) when is_map(progress) do + percent = Map.get(progress, :percent, Map.get(progress, "percent")) + bytes_sent = Map.get(progress, :bytes_sent, Map.get(progress, "bytes_sent")) + total_bytes = Map.get(progress, :total_bytes, Map.get(progress, "total_bytes")) + + "#{label}(percent=#{percent || "?"}, bytes_sent=#{bytes_sent || "?"}, total_bytes=#{total_bytes || "?"})" + end + defp video_active_in_state?({:error, _}, _video_id), do: false defp video_active_in_state?(state, video_id) when is_map(state) do diff --git a/lib/reencodarr_web/channels/worker_channel.ex b/lib/reencodarr_web/channels/worker_channel.ex index 0c93e782..755c054e 100644 --- a/lib/reencodarr_web/channels/worker_channel.ex +++ b/lib/reencodarr_web/channels/worker_channel.ex @@ -136,6 +136,7 @@ defmodule ReencodarrWeb.WorkerChannel do defp handle_transfer_progress(payload, %{assigns: %{worker_id: worker_id}} = socket) do with {:ok, progress} <- WorkerProtocol.parse_transfer_progress(payload), :ok <- ensure_active_video(socket, progress.video_id) do + _ = WorkerSessions.set_transfer_progress(worker_id, progress) Events.broadcast_event(:transfer_progress, Map.put(progress, :worker_id, worker_id)) {:reply, {:ok, WorkerProtocol.event_ack("transfer_progress")}, socket} else @@ -147,6 +148,7 @@ defmodule ReencodarrWeb.WorkerChannel do defp handle_crf_search_progress(payload, socket) do with {:ok, progress} <- WorkerProtocol.parse_crf_search_progress(payload), :ok <- ensure_active_video(socket, progress.video_id) do + _ = WorkerSessions.set_crf_search_progress(socket.assigns.worker_id, progress) Events.broadcast_event(:crf_search_progress, progress) {:reply, {:ok, WorkerProtocol.event_ack("crf_search_progress")}, socket} else diff --git a/test/reencodarr/ab_av1/worker_config_test.exs b/test/reencodarr/ab_av1/worker_config_test.exs new file mode 100644 index 00000000..8b0ac16a --- /dev/null +++ b/test/reencodarr/ab_av1/worker_config_test.exs @@ -0,0 +1,23 @@ +defmodule Reencodarr.AbAv1.WorkerConfigTest do + use ExUnit.Case, async: false + + alias Reencodarr.AbAv1.WorkerConfig + + test "can enable and disable distributed workers" do + previous = Application.get_env(:reencodarr, :distributed_worker_enabled) + + try do + assert :ok = WorkerConfig.enable() + assert WorkerConfig.enabled?() + + assert :ok = WorkerConfig.disable() + refute WorkerConfig.enabled?() + after + if is_nil(previous) do + Application.delete_env(:reencodarr, :distributed_worker_enabled) + else + Application.put_env(:reencodarr, :distributed_worker_enabled, previous) + end + end + end +end diff --git a/test/reencodarr/ab_av1/worker_sessions_test.exs b/test/reencodarr/ab_av1/worker_sessions_test.exs index 889d4f7f..5807feea 100644 --- a/test/reencodarr/ab_av1/worker_sessions_test.exs +++ b/test/reencodarr/ab_av1/worker_sessions_test.exs @@ -5,6 +5,7 @@ defmodule Reencodarr.AbAv1.WorkerSessionsTest do alias Reencodarr.Dashboard.Events alias Reencodarr.Diagnostics alias Reencodarr.Fixtures + alias Reencodarr.Media setup do WorkerSessions.reset() @@ -75,6 +76,33 @@ defmodule Reencodarr.AbAv1.WorkerSessionsTest do assert listed_session.active_video_id == 123 end + test "cancels and drains active distributed work" do + {:ok, video_one} = Fixtures.video_fixture(%{state: :analyzed}) + {:ok, video_two} = Fixtures.video_fixture(%{state: :analyzed}) + + assert {:ok, _session} = + WorkerSessions.register(worker_session_attrs(server_worker_id: "worker-server-1")) + + assert {:ok, _session} = + WorkerSessions.register( + worker_session_attrs( + server_worker_id: "worker-server-2", + client_worker_id: "worker-client-2" + ) + ) + + assert {:ok, _session} = WorkerSessions.assign_video("worker-server-1", video_one.id) + assert {:ok, _session} = WorkerSessions.assign_video("worker-server-2", video_two.id) + + assert {:ok, _session} = WorkerSessions.cancel("worker-server-1") + assert Media.get_video(video_one.id).state == :analyzed + + assert {:ok, drained_sessions} = WorkerSessions.drain() + assert Enum.map(drained_sessions, & &1.server_worker_id) == ["worker-server-2"] + assert Media.get_video(video_two.id).state == :analyzed + assert WorkerSessions.list() == [] + end + defp worker_session_attrs(overrides \\ []) do %{ server_worker_id: "worker-server-1", diff --git a/test/reencodarr_web/channels/worker_channel_test.exs b/test/reencodarr_web/channels/worker_channel_test.exs index cfcc2044..fa28a906 100644 --- a/test/reencodarr_web/channels/worker_channel_test.exs +++ b/test/reencodarr_web/channels/worker_channel_test.exs @@ -203,6 +203,7 @@ defmodule ReencodarrWeb.WorkerChannelTest do assert {:ok, socket} = connect(WorkerSocket, %{"token" => token}) assert {:ok, _join_payload, socket} = subscribe_and_join(socket, "workers:crf_search") + server_worker_id = socket.assigns.worker_id assert_reply push(socket, "announce", announce_payload(worker_id: "worker-a")), :ok, @@ -253,6 +254,9 @@ defmodule ReencodarrWeb.WorkerChannelTest do }} assert filename == Path.basename(video.path) + session = WorkerSessions.get(server_worker_id) + assert session.transfer_progress.percent == 25.5 + assert session.crf_search_progress.percent == 62.0 after Application.delete_env(:reencodarr, :worker_token) end From c8a6736f2901272389e4b6cca94d1ca9b6d9182f Mon Sep 17 00:00:00 2001 From: Mika Cohen Date: Tue, 7 Jul 2026 14:47:34 -0600 Subject: [PATCH 27/70] Show worker token on dashboard --- lib/reencodarr_web/live/dashboard_live.ex | 16 ++++++++++++++-- .../live/dashboard_v2_live_test.exs | 1 + 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/lib/reencodarr_web/live/dashboard_live.ex b/lib/reencodarr_web/live/dashboard_live.ex index 898f72d8..0b54ebe0 100644 --- a/lib/reencodarr_web/live/dashboard_live.ex +++ b/lib/reencodarr_web/live/dashboard_live.ex @@ -1050,11 +1050,23 @@ defmodule ReencodarrWeb.DashboardLive do +
+
Token
+
+ <%= case @worker_token_state do %> + <% {:ok, token, _fingerprint} -> %> + {token} + <% :error -> %> + not configured + <% end %> +
+
+
Token fingerprint
<%= case @worker_token_state do %> - <% {:ok, fingerprint} -> %> + <% {:ok, _token, fingerprint} -> %> {fingerprint} <% :error -> %> not configured @@ -1100,7 +1112,7 @@ defmodule ReencodarrWeb.DashboardLive do |> Base.encode16(case: :lower) |> String.slice(0, 12) - {:ok, "sha256:#{digest}"} + {:ok, token, "sha256:#{digest}"} _ -> :error diff --git a/test/reencodarr_web/live/dashboard_v2_live_test.exs b/test/reencodarr_web/live/dashboard_v2_live_test.exs index 6f7195d6..19ccd0d5 100644 --- a/test/reencodarr_web/live/dashboard_v2_live_test.exs +++ b/test/reencodarr_web/live/dashboard_v2_live_test.exs @@ -99,6 +99,7 @@ defmodule ReencodarrWeb.DashboardLiveTest do expected_fingerprint = worker_token_fingerprint("deploy-test-worker-token") assert html =~ "Worker WebSocket" + assert html =~ "deploy-test-worker-token" assert html =~ expected_fingerprint assert html =~ "/workers/socket/websocket?token=" end From e9451423457a332ae03251b3242fe7b8343a98a3 Mon Sep 17 00:00:00 2001 From: Mika Cohen Date: Tue, 7 Jul 2026 15:31:06 -0600 Subject: [PATCH 28/70] Enable worker socket by default --- config/config.exs | 2 +- config/runtime.exs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/config/config.exs b/config/config.exs index dea90bd7..cb291da6 100644 --- a/config/config.exs +++ b/config/config.exs @@ -13,7 +13,7 @@ config :reencodarr, env: config_env(), dashboard_stats_query_timeout_ms: 500, dashboard_queue_query_timeout_ms: 5_000, - distributed_worker_enabled: false, + distributed_worker_enabled: true, worker_chunk_size_bytes: 1_048_576, worker_transfer_window: 1, worker_retry_limit: 3, diff --git a/config/runtime.exs b/config/runtime.exs index 4eafd4ee..59b0766c 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -57,7 +57,7 @@ end config :reencodarr, distributed_worker_enabled: - parse_bool_env.(System.get_env("REENCODARR_DISTRIBUTED_WORKERS", "false")), + parse_bool_env.(System.get_env("REENCODARR_DISTRIBUTED_WORKERS", "true")), worker_chunk_size_bytes: parse_int_env.("REENCODARR_WORKER_CHUNK_SIZE_BYTES", 1_048_576), worker_transfer_window: parse_int_env.("REENCODARR_WORKER_TRANSFER_WINDOW", 1), worker_retry_limit: parse_int_env.("REENCODARR_WORKER_RETRY_LIMIT", 3), From 7f80c38b3db701044edb1148d4d963e87103970d Mon Sep 17 00:00:00 2001 From: Mika Cohen Date: Tue, 7 Jul 2026 17:29:57 -0600 Subject: [PATCH 29/70] Use binary frames for worker transfers --- config/config.exs | 2 +- config/runtime.exs | 2 +- docs/distributed_worker_protocol.md | 31 ++++--- lib/reencodarr/ab_av1/worker_config.ex | 2 +- lib/reencodarr/ab_av1/worker_protocol.ex | 87 ++++++++++++++++--- .../ab_av1/worker_protocol_test.exs | 31 ++++++- .../channels/worker_channel_test.exs | 76 +++++++++------- 7 files changed, 171 insertions(+), 60 deletions(-) diff --git a/config/config.exs b/config/config.exs index cb291da6..164455f7 100644 --- a/config/config.exs +++ b/config/config.exs @@ -14,7 +14,7 @@ config :reencodarr, dashboard_stats_query_timeout_ms: 500, dashboard_queue_query_timeout_ms: 5_000, distributed_worker_enabled: true, - worker_chunk_size_bytes: 1_048_576, + worker_chunk_size_bytes: 134_217_728, worker_transfer_window: 1, worker_retry_limit: 3, worker_transfer_timeout_ms: 60_000, diff --git a/config/runtime.exs b/config/runtime.exs index 59b0766c..da15f2ee 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -58,7 +58,7 @@ end config :reencodarr, distributed_worker_enabled: parse_bool_env.(System.get_env("REENCODARR_DISTRIBUTED_WORKERS", "true")), - worker_chunk_size_bytes: parse_int_env.("REENCODARR_WORKER_CHUNK_SIZE_BYTES", 1_048_576), + worker_chunk_size_bytes: parse_int_env.("REENCODARR_WORKER_CHUNK_SIZE_BYTES", 134_217_728), worker_transfer_window: parse_int_env.("REENCODARR_WORKER_TRANSFER_WINDOW", 1), worker_retry_limit: parse_int_env.("REENCODARR_WORKER_RETRY_LIMIT", 3), worker_transfer_timeout_ms: parse_int_env.("REENCODARR_WORKER_TRANSFER_TIMEOUT_MS", 60_000), diff --git a/docs/distributed_worker_protocol.md b/docs/distributed_worker_protocol.md index ecfbb842..7dc874ae 100644 --- a/docs/distributed_worker_protocol.md +++ b/docs/distributed_worker_protocol.md @@ -41,7 +41,7 @@ Server reply: - `chunk_size_bytes` - `target_vmaf` -The current chunk size is `1_048_576` bytes. +The default chunk size is `134_217_728` bytes (128 MiB). ## Media Transfer @@ -62,23 +62,29 @@ Fields: ### `transfer_chunk` -Fields: +`transfer_chunk` is a binary Phoenix channel payload. It is not JSON and does +not base64-encode media bytes. -- `status: "transfer_chunk"` -- `video_id` -- `transfer_id` -- `chunk_index` -- `total_chunks` -- `bytes_sent` -- `total_bytes` -- `crc32` -- `data` +Frame layout: + +- `magic`: 4 bytes, `RAV1` +- `version`: 1 byte, currently `1` +- `type`: 1 byte, currently `1` for transfer chunk +- `transfer_id_size`: 16-bit unsigned integer +- `video_id`: 64-bit unsigned integer +- `chunk_index`: 64-bit unsigned integer +- `total_chunks`: 64-bit unsigned integer +- `bytes_sent`: 64-bit unsigned integer +- `total_bytes`: 64-bit unsigned integer +- `crc32`: 32-bit unsigned integer +- `transfer_id`: `transfer_id_size` bytes +- `data`: raw chunk bytes Notes: - `chunk_index` starts at `0` and increments monotonically. - `crc32` is computed from the raw chunk bytes. -- `data` is base64-encoded chunk data. +- `data` is the raw file data for that chunk. ### `transfer_complete` @@ -118,4 +124,3 @@ The channel also accepts these client-to-server events: - `crf_search_completed` with `cancelled` or `shutdown` returns the video to `analyzed`. - `video_failed` records a structured failure and clears the active assignment. - Disconnecting a worker with active CRF-search work requeues the video. - diff --git a/lib/reencodarr/ab_av1/worker_config.ex b/lib/reencodarr/ab_av1/worker_config.ex index 62485d44..60525940 100644 --- a/lib/reencodarr/ab_av1/worker_config.ex +++ b/lib/reencodarr/ab_av1/worker_config.ex @@ -3,7 +3,7 @@ defmodule Reencodarr.AbAv1.WorkerConfig do Runtime configuration for distributed ab-av1 workers. """ - @default_chunk_size_bytes 1_048_576 + @default_chunk_size_bytes 134_217_728 @spec enabled?() :: boolean() def enabled? do diff --git a/lib/reencodarr/ab_av1/worker_protocol.ex b/lib/reencodarr/ab_av1/worker_protocol.ex index bfa89c78..13e59435 100644 --- a/lib/reencodarr/ab_av1/worker_protocol.ex +++ b/lib/reencodarr/ab_av1/worker_protocol.ex @@ -8,6 +8,9 @@ defmodule Reencodarr.AbAv1.WorkerProtocol do @crf_search_topic "workers:crf_search" @supported_protocol_versions [1] + @transfer_chunk_magic "RAV1" + @transfer_chunk_frame_version 1 + @transfer_chunk_frame_type 1 defmodule Announcement do @moduledoc false @@ -304,7 +307,7 @@ defmodule Reencodarr.AbAv1.WorkerProtocol do non_neg_integer(), non_neg_integer(), binary() - ) :: map() + ) :: {:binary, binary()} def transfer_chunk( %Video{id: video_id}, transfer_id, @@ -317,17 +320,77 @@ defmodule Reencodarr.AbAv1.WorkerProtocol do when is_integer(video_id) and is_binary(transfer_id) and is_integer(chunk_index) and is_integer(total_chunks) and is_integer(bytes_sent) and is_integer(total_bytes) and is_binary(chunk) do - %{ - status: "transfer_chunk", - video_id: video_id, - transfer_id: transfer_id, - chunk_index: chunk_index, - total_chunks: total_chunks, - bytes_sent: bytes_sent, - total_bytes: total_bytes, - crc32: :erlang.crc32(chunk), - data: Base.encode64(chunk) - } + {:binary, + transfer_chunk_frame( + video_id, + transfer_id, + chunk_index, + total_chunks, + bytes_sent, + total_bytes, + chunk + )} + end + + @spec parse_transfer_chunk_frame(binary()) :: {:ok, map()} | {:error, atom()} + def parse_transfer_chunk_frame( + <<@transfer_chunk_magic, @transfer_chunk_frame_version, @transfer_chunk_frame_type, + transfer_id_size::16, video_id::64, chunk_index::64, total_chunks::64, bytes_sent::64, + total_bytes::64, crc32::32, transfer_id::binary-size(transfer_id_size), data::binary>> + ) do + if :erlang.crc32(data) == crc32 do + {:ok, + %{ + video_id: video_id, + transfer_id: transfer_id, + chunk_index: chunk_index, + total_chunks: total_chunks, + bytes_sent: bytes_sent, + total_bytes: total_bytes, + crc32: crc32, + data: data + }} + else + {:error, :crc_mismatch} + end + end + + def parse_transfer_chunk_frame( + <<@transfer_chunk_magic, @transfer_chunk_frame_version, _frame_type, _rest::binary>> + ), + do: {:error, :unsupported_transfer_chunk_frame_type} + + def parse_transfer_chunk_frame(<<@transfer_chunk_magic, _version, _rest::binary>>), + do: {:error, :unsupported_transfer_chunk_frame_version} + + def parse_transfer_chunk_frame(_frame), do: {:error, :invalid_transfer_chunk_frame} + + defp transfer_chunk_frame( + video_id, + transfer_id, + chunk_index, + total_chunks, + bytes_sent, + total_bytes, + chunk + ) do + transfer_id_size = byte_size(transfer_id) + crc32 = :erlang.crc32(chunk) + + << + @transfer_chunk_magic, + @transfer_chunk_frame_version, + @transfer_chunk_frame_type, + transfer_id_size::16, + video_id::64, + chunk_index::64, + total_chunks::64, + bytes_sent::64, + total_bytes::64, + crc32::32, + transfer_id::binary, + chunk::binary + >> end @spec transfer_complete(Video.t(), String.t(), non_neg_integer(), non_neg_integer()) :: map() diff --git a/test/reencodarr/ab_av1/worker_protocol_test.exs b/test/reencodarr/ab_av1/worker_protocol_test.exs index 62547c98..d7d3883a 100644 --- a/test/reencodarr/ab_av1/worker_protocol_test.exs +++ b/test/reencodarr/ab_av1/worker_protocol_test.exs @@ -177,11 +177,40 @@ defmodule Reencodarr.AbAv1.WorkerProtocolTest do video_id: 123, source_name: "movie.mkv", size_bytes: 987_654, - chunk_size_bytes: 1_048_576, + chunk_size_bytes: 134_217_728, target_vmaf: 96.5 } end + test "builds binary transfer chunk frames with ordered metadata and raw data" do + chunk = "raw video bytes" + + assert {:binary, frame} = + WorkerProtocol.transfer_chunk( + %Reencodarr.Media.Video{id: 123}, + "transfer-123", + 7, + 9, + 1_024, + 2_048, + chunk + ) + + assert {:ok, + %{ + video_id: 123, + transfer_id: "transfer-123", + chunk_index: 7, + total_chunks: 9, + bytes_sent: 1_024, + total_bytes: 2_048, + crc32: crc32, + data: ^chunk + }} = WorkerProtocol.parse_transfer_chunk_frame(frame) + + assert crc32 == :erlang.crc32(chunk) + end + test "reads chunk size from configuration" do previous = Application.get_env(:reencodarr, :worker_chunk_size_bytes) Application.put_env(:reencodarr, :worker_chunk_size_bytes, 2_097_152) diff --git a/test/reencodarr_web/channels/worker_channel_test.exs b/test/reencodarr_web/channels/worker_channel_test.exs index fa28a906..c47c0428 100644 --- a/test/reencodarr_web/channels/worker_channel_test.exs +++ b/test/reencodarr_web/channels/worker_channel_test.exs @@ -1,6 +1,7 @@ defmodule ReencodarrWeb.WorkerChannelTest do use ReencodarrWeb.ChannelCase, async: false + alias Reencodarr.AbAv1.WorkerProtocol alias Reencodarr.AbAv1.WorkerSessions alias Reencodarr.Dashboard.Events alias Reencodarr.Fixtures @@ -73,7 +74,7 @@ defmodule ReencodarrWeb.WorkerChannelTest do video_id: assigned_video_id, source_name: source_name, size_bytes: size_bytes, - chunk_size_bytes: 1_048_576 + chunk_size_bytes: 134_217_728 } assert job_id == Integer.to_string(video.id) @@ -263,16 +264,25 @@ defmodule ReencodarrWeb.WorkerChannelTest do test "streams assigned media in ordered chunks with integrity data" do token = "test-worker-token" + previous_chunk_size = Application.get_env(:reencodarr, :worker_chunk_size_bytes) Application.put_env(:reencodarr, :worker_token, token) + Application.put_env(:reencodarr, :worker_chunk_size_bytes, 4) - content = :binary.copy("abcd", 262_145) + on_exit(fn -> + if is_nil(previous_chunk_size) do + Application.delete_env(:reencodarr, :worker_chunk_size_bytes) + else + Application.put_env(:reencodarr, :worker_chunk_size_bytes, previous_chunk_size) + end + end) + + chunk_size = 4 + content = "abcdefgh" content_size = byte_size(content) - first_chunk = binary_part(content, 0, 1_048_576) - second_chunk = binary_part(content, 1_048_576, content_size - 1_048_576) + first_chunk = binary_part(content, 0, chunk_size) + second_chunk = binary_part(content, chunk_size, content_size - chunk_size) first_crc32 = :erlang.crc32(first_chunk) second_crc32 = :erlang.crc32(second_chunk) - first_encoded = Base.encode64(first_chunk) - second_encoded = Base.encode64(second_chunk) with_temp_file(content, ".mkv", fn path -> {:ok, video} = @@ -294,7 +304,7 @@ defmodule ReencodarrWeb.WorkerChannelTest do %{ status: "job_assigned", video_id: assigned_video_id, - chunk_size_bytes: 1_048_576 + chunk_size_bytes: ^chunk_size } assert assigned_video_id == video.id @@ -304,34 +314,38 @@ defmodule ReencodarrWeb.WorkerChannelTest do video_id: ^assigned_video_id, transfer_id: transfer_id, size_bytes: content_size, - chunk_size_bytes: 1_048_576, + chunk_size_bytes: ^chunk_size, total_bytes: content_size, total_chunks: 2 } - assert_push "transfer_chunk", %{ - status: "transfer_chunk", - video_id: ^assigned_video_id, - transfer_id: ^transfer_id, - chunk_index: 0, - total_chunks: 2, - bytes_sent: 1_048_576, - total_bytes: ^content_size, - crc32: ^first_crc32, - data: ^first_encoded - } - - assert_push "transfer_chunk", %{ - status: "transfer_chunk", - video_id: ^assigned_video_id, - transfer_id: ^transfer_id, - chunk_index: 1, - total_chunks: 2, - bytes_sent: ^content_size, - total_bytes: ^content_size, - crc32: ^second_crc32, - data: ^second_encoded - } + assert_push "transfer_chunk", {:binary, first_frame} + + assert {:ok, + %{ + video_id: ^assigned_video_id, + transfer_id: ^transfer_id, + chunk_index: 0, + total_chunks: 2, + bytes_sent: ^chunk_size, + total_bytes: ^content_size, + crc32: ^first_crc32, + data: ^first_chunk + }} = WorkerProtocol.parse_transfer_chunk_frame(first_frame) + + assert_push "transfer_chunk", {:binary, second_frame} + + assert {:ok, + %{ + video_id: ^assigned_video_id, + transfer_id: ^transfer_id, + chunk_index: 1, + total_chunks: 2, + bytes_sent: ^content_size, + total_bytes: ^content_size, + crc32: ^second_crc32, + data: ^second_chunk + }} = WorkerProtocol.parse_transfer_chunk_frame(second_frame) assert_push "transfer_complete", %{ status: "transfer_complete", From c0824e619ef36a483b3fa082dc8745858142a3ab Mon Sep 17 00:00:00 2001 From: Mika Cohen Date: Tue, 7 Jul 2026 18:13:41 -0600 Subject: [PATCH 30/70] Show worker progress telemetry --- lib/reencodarr/ab_av1/worker_protocol.ex | 43 +++++++ lib/reencodarr/ab_av1/worker_sessions.ex | 10 +- lib/reencodarr_web/channels/worker_channel.ex | 4 +- lib/reencodarr_web/live/workers_live.ex | 111 +++++++++++++++++- .../ab_av1/worker_protocol_test.exs | 19 +++ .../channels/worker_channel_test.exs | 21 +++- .../reencodarr_web/live/workers_live_test.exs | 39 ++++++ 7 files changed, 236 insertions(+), 11 deletions(-) diff --git a/lib/reencodarr/ab_av1/worker_protocol.ex b/lib/reencodarr/ab_av1/worker_protocol.ex index 13e59435..6f751f9c 100644 --- a/lib/reencodarr/ab_av1/worker_protocol.ex +++ b/lib/reencodarr/ab_av1/worker_protocol.ex @@ -440,6 +440,49 @@ defmodule Reencodarr.AbAv1.WorkerProtocol do def heartbeat_ack(last_seen_at), do: %{accepted: true, last_seen_at: DateTime.to_iso8601(last_seen_at)} + @spec parse_resource_usage(map()) :: map() | nil + def parse_resource_usage(payload) when is_map(payload) do + [ + cpu_percent: optional_number(payload, [:cpu_percent, "cpu_percent"]), + memory_bytes: + optional_integer(payload, [ + :memory_bytes, + "memory_bytes", + :memory_rss_bytes, + "memory_rss_bytes" + ]), + memory_total_bytes: + optional_integer(payload, [ + :memory_total_bytes, + "memory_total_bytes", + :total_memory_bytes, + "total_memory_bytes" + ]), + disk_free_bytes: + optional_integer(payload, [ + :disk_free_bytes, + "disk_free_bytes", + :free_disk_bytes, + "free_disk_bytes" + ]), + disk_total_bytes: + optional_integer(payload, [ + :disk_total_bytes, + "disk_total_bytes", + :total_disk_bytes, + "total_disk_bytes" + ]) + ] + |> Enum.reject(fn {_key, value} -> is_nil(value) end) + |> Map.new() + |> case do + usage when usage == %{} -> nil + usage -> usage + end + end + + def parse_resource_usage(_payload), do: nil + @spec error( :duplicate_worker_id | :invalid_announcement diff --git a/lib/reencodarr/ab_av1/worker_sessions.ex b/lib/reencodarr/ab_av1/worker_sessions.ex index f9d50627..6545d867 100644 --- a/lib/reencodarr/ab_av1/worker_sessions.ex +++ b/lib/reencodarr/ab_av1/worker_sessions.ex @@ -19,6 +19,7 @@ defmodule Reencodarr.AbAv1.WorkerSessions do active_video_id: integer() | nil, transfer_progress: map() | nil, crf_search_progress: map() | nil, + resource_usage: map() | nil, connected_at: DateTime.t(), last_seen_at: DateTime.t() } @@ -31,8 +32,8 @@ defmodule Reencodarr.AbAv1.WorkerSessions do GenServer.call(__MODULE__, {:register, attrs}) end - def touch(server_worker_id) do - GenServer.call(__MODULE__, {:touch, server_worker_id}) + def touch(server_worker_id, resource_usage \\ nil) do + GenServer.call(__MODULE__, {:touch, server_worker_id, resource_usage}) end def unregister(server_worker_id) do @@ -111,9 +112,9 @@ defmodule Reencodarr.AbAv1.WorkerSessions do end end - def handle_call({:touch, server_worker_id}, _from, state) do + def handle_call({:touch, server_worker_id, resource_usage}, _from, state) do update_session_reply(server_worker_id, state, fn session -> - %{session | last_seen_at: now()} + %{session | last_seen_at: now(), resource_usage: resource_usage || session.resource_usage} end) end @@ -231,6 +232,7 @@ defmodule Reencodarr.AbAv1.WorkerSessions do active_video_id: nil, transfer_progress: nil, crf_search_progress: nil, + resource_usage: nil, connected_at: now, last_seen_at: now } diff --git a/lib/reencodarr_web/channels/worker_channel.ex b/lib/reencodarr_web/channels/worker_channel.ex index 755c054e..90265718 100644 --- a/lib/reencodarr_web/channels/worker_channel.ex +++ b/lib/reencodarr_web/channels/worker_channel.ex @@ -51,8 +51,8 @@ defmodule ReencodarrWeb.WorkerChannel do end end - def handle_in("heartbeat", _payload, %{assigns: %{worker_id: worker_id}} = socket) do - case WorkerSessions.touch(worker_id) do + def handle_in("heartbeat", payload, %{assigns: %{worker_id: worker_id}} = socket) do + case WorkerSessions.touch(worker_id, WorkerProtocol.parse_resource_usage(payload)) do {:ok, session} -> {:reply, {:ok, WorkerProtocol.heartbeat_ack(session.last_seen_at)}, socket} diff --git a/lib/reencodarr_web/live/workers_live.ex b/lib/reencodarr_web/live/workers_live.ex index 5a837e1a..927906f2 100644 --- a/lib/reencodarr_web/live/workers_live.ex +++ b/lib/reencodarr_web/live/workers_live.ex @@ -7,6 +7,7 @@ defmodule ReencodarrWeb.WorkersLive do alias Reencodarr.AbAv1.WorkerSessions alias Reencodarr.Dashboard.Events + alias Reencodarr.Formatters alias Reencodarr.Media @refresh_interval 5_000 @@ -70,6 +71,9 @@ defmodule ReencodarrWeb.WorkersLive do Worker State Video + Resources + Live Progress + CRF/VMAF Results Protocol Version Connected @@ -79,7 +83,7 @@ defmodule ReencodarrWeb.WorkersLive do <%= if @workers == [] do %> - + No workers connected. @@ -101,6 +105,18 @@ defmodule ReencodarrWeb.WorkersLive do {worker_video(worker)} + +
CPU {worker_cpu(worker)}
+
Mem {worker_memory(worker)}
+
Disk {worker_disk(worker)}
+ + +
{worker_progress(worker)}
+
{worker_transfer(worker)}
+ + + {worker_vmafs(worker)} + {worker.protocol_version} @@ -135,6 +151,99 @@ defmodule ReencodarrWeb.WorkersLive do defp worker_video(%{active_video_id: nil}), do: "none" defp worker_video(%{active_video_id: video_id}), do: "video ##{video_id}" + defp worker_cpu(%{resource_usage: %{cpu_percent: cpu_percent}}) when is_number(cpu_percent), + do: "#{format_number(cpu_percent)}%" + + defp worker_cpu(_worker), do: "-" + + defp worker_memory(%{resource_usage: %{memory_bytes: memory_bytes} = usage}) + when is_integer(memory_bytes) do + case Map.get(usage, :memory_total_bytes) do + total when is_integer(total) and total > 0 -> + "#{Formatters.file_size(memory_bytes)} / #{Formatters.file_size(total)}" + + _ -> + Formatters.file_size(memory_bytes) + end + end + + defp worker_memory(_worker), do: "-" + + defp worker_disk(%{resource_usage: %{disk_free_bytes: disk_free_bytes} = usage}) + when is_integer(disk_free_bytes) do + case Map.get(usage, :disk_total_bytes) do + total when is_integer(total) and total > 0 -> + "#{Formatters.file_size(disk_free_bytes)} free / #{Formatters.file_size(total)}" + + _ -> + "#{Formatters.file_size(disk_free_bytes)} free" + end + end + + defp worker_disk(_worker), do: "-" + + defp worker_progress(%{crf_search_progress: nil}), do: "CRF -" + + defp worker_progress(%{crf_search_progress: progress}) do + [ + {"CRF", progress.percent, &"#{format_number(&1)}%"}, + {"FPS", progress.fps, &Formatters.fps/1}, + {"ETA", progress.eta, &"#{&1}s"} + ] + |> format_parts() + end + + defp worker_transfer(%{transfer_progress: nil}), do: "Transfer -" + + defp worker_transfer(%{transfer_progress: progress}) do + [ + {"Transfer", progress.percent, &"#{format_number(&1)}%"}, + {"Chunk", progress.chunk_index, &to_string/1}, + {"Total", progress.total_chunks, &to_string/1} + ] + |> format_parts() + end + + defp worker_vmafs(%{active_video_id: nil}), do: "-" + + defp worker_vmafs(%{active_video_id: video_id}) do + video_id + |> Media.get_vmafs_for_video() + |> Enum.sort_by(& &1.crf) + |> Enum.take(4) + |> case do + [] -> + "-" + + vmafs -> + Enum.map_join(vmafs, " / ", &format_vmaf/1) + end + end + + defp format_vmaf(vmaf) do + "CRF #{Formatters.crf(vmaf.crf)} -> #{Formatters.vmaf_score(vmaf.score, 1)} (#{format_number(vmaf.percent)}%)" + end + + defp format_parts(parts) do + parts + |> Enum.reject(fn {_label, value, _formatter} -> is_nil(value) end) + |> Enum.map(fn {label, value, formatter} -> + "#{label} #{formatter.(value)}" + end) + |> case do + [] -> "-" + formatted -> Enum.join(formatted, " / ") + end + end + + defp format_number(number) when is_integer(number), do: Integer.to_string(number) + + defp format_number(number) when is_float(number) do + :erlang.float_to_binary(number, decimals: 1) + end + + defp format_number(nil), do: "-" + defp status_badge_class(%{active_video_id: nil}), do: "inline-flex rounded-full border border-emerald-900 bg-emerald-950 px-2 py-1 text-xs font-medium text-emerald-300" diff --git a/test/reencodarr/ab_av1/worker_protocol_test.exs b/test/reencodarr/ab_av1/worker_protocol_test.exs index d7d3883a..43c8e865 100644 --- a/test/reencodarr/ab_av1/worker_protocol_test.exs +++ b/test/reencodarr/ab_av1/worker_protocol_test.exs @@ -77,6 +77,25 @@ defmodule Reencodarr.AbAv1.WorkerProtocolTest do }) end + test "parses heartbeat resource usage" do + assert %{ + cpu_percent: 87.5, + memory_bytes: 1_073_741_824, + memory_total_bytes: 4_294_967_296, + disk_free_bytes: 536_870_912_000, + disk_total_bytes: 1_099_511_627_776 + } = + WorkerProtocol.parse_resource_usage(%{ + "cpu_percent" => 87.5, + "memory_rss_bytes" => 1_073_741_824, + "total_memory_bytes" => 4_294_967_296, + "free_disk_bytes" => 536_870_912_000, + "total_disk_bytes" => 1_099_511_627_776 + }) + + assert is_nil(WorkerProtocol.parse_resource_usage(%{})) + end + test "parses structured CRF results and completion payloads" do assert {:ok, %CrfSearchResult{ diff --git a/test/reencodarr_web/channels/worker_channel_test.exs b/test/reencodarr_web/channels/worker_channel_test.exs index c47c0428..0469f7e6 100644 --- a/test/reencodarr_web/channels/worker_channel_test.exs +++ b/test/reencodarr_web/channels/worker_channel_test.exs @@ -179,10 +179,18 @@ defmodule ReencodarrWeb.WorkerChannelTest do connected_at = session.connected_at first_seen_at = session.last_seen_at - assert_reply push(socket, "heartbeat", %{}), :ok, %{ - accepted: true, - last_seen_at: last_seen_at - } + assert_reply push(socket, "heartbeat", %{ + "cpu_percent" => 87.5, + "memory_bytes" => 1_073_741_824, + "memory_total_bytes" => 4_294_967_296, + "disk_free_bytes" => 536_870_912_000, + "disk_total_bytes" => 1_099_511_627_776 + }), + :ok, + %{ + accepted: true, + last_seen_at: last_seen_at + } assert {:ok, parsed_last_seen_at, 0} = DateTime.from_iso8601(last_seen_at) @@ -190,6 +198,11 @@ defmodule ReencodarrWeb.WorkerChannelTest do assert updated_session.connected_at == connected_at assert DateTime.compare(updated_session.last_seen_at, first_seen_at) in [:eq, :gt] assert updated_session.last_seen_at == parsed_last_seen_at + assert updated_session.resource_usage.cpu_percent == 87.5 + assert updated_session.resource_usage.memory_bytes == 1_073_741_824 + assert updated_session.resource_usage.memory_total_bytes == 4_294_967_296 + assert updated_session.resource_usage.disk_free_bytes == 536_870_912_000 + assert updated_session.resource_usage.disk_total_bytes == 1_099_511_627_776 after Application.delete_env(:reencodarr, :worker_token) end diff --git a/test/reencodarr_web/live/workers_live_test.exs b/test/reencodarr_web/live/workers_live_test.exs index 6efa66ec..61e38dc7 100644 --- a/test/reencodarr_web/live/workers_live_test.exs +++ b/test/reencodarr_web/live/workers_live_test.exs @@ -37,9 +37,48 @@ defmodule ReencodarrWeb.WorkersLiveTest do assert {:ok, _session} = WorkerSessions.assign_video("worker-server-1", video.id) + assert {:ok, _session} = + WorkerSessions.touch("worker-server-1", %{ + cpu_percent: 87.5, + memory_bytes: 1_073_741_824, + memory_total_bytes: 4_294_967_296, + disk_free_bytes: 536_870_912_000, + disk_total_bytes: 1_099_511_627_776 + }) + + assert {:ok, _session} = + WorkerSessions.set_crf_search_progress("worker-server-1", %{ + percent: 62.0, + fps: 12.5, + eta: 90 + }) + + assert {:ok, _session} = + WorkerSessions.set_transfer_progress("worker-server-1", %{ + percent: 25.5, + chunk_index: 2, + total_chunks: 8 + }) + + Fixtures.vmaf_fixture(%{ + video_id: video.id, + crf: 28.0, + score: 95.4, + percent: 93.0 + }) + send(view.pid, {:worker_sessions_updated, %{sessions: WorkerSessions.list()}}) html = render(view) assert html =~ "analyzed" assert html =~ "video ##{video.id}" + assert html =~ "CPU 87.5%" + assert html =~ "Mem 1.0 GiB / 4.0 GiB" + assert html =~ "Disk 500.0 GiB free / 1.0 TiB" + assert html =~ "CRF 62.0%" + assert html =~ "FPS 12.5 fps" + assert html =~ "ETA 90s" + assert html =~ "Transfer 25.5%" + assert html =~ "Chunk 2" + assert html =~ "CRF 28.0 -> 95.4 (93.0%)" end end From addaf4794b29c48f127df49107fd435915d8570f Mon Sep 17 00:00:00 2001 From: Mika Cohen Date: Tue, 7 Jul 2026 18:56:22 -0600 Subject: [PATCH 31/70] Accept worker transfer progress --- lib/reencodarr/ab_av1/worker_protocol.ex | 32 +++++++++++++++-- lib/reencodarr_web/live/workers_live.ex | 36 +++++++++++++++++-- .../ab_av1/worker_protocol_test.exs | 15 +++++--- .../channels/worker_channel_test.exs | 21 ++++++++--- .../reencodarr_web/live/workers_live_test.exs | 10 ++++++ 5 files changed, 101 insertions(+), 13 deletions(-) diff --git a/lib/reencodarr/ab_av1/worker_protocol.ex b/lib/reencodarr/ab_av1/worker_protocol.ex index 6f751f9c..0537f5e9 100644 --- a/lib/reencodarr/ab_av1/worker_protocol.ex +++ b/lib/reencodarr/ab_av1/worker_protocol.ex @@ -31,21 +31,29 @@ defmodule Reencodarr.AbAv1.WorkerProtocol do @enforce_keys [:video_id, :transfer_id, :percent, :bytes_sent, :total_bytes] defstruct [ + :job_id, :video_id, :transfer_id, + :filename, :percent, :bytes_sent, :total_bytes, + :bytes_per_second, + :eta, :chunk_index, :total_chunks ] @type t :: %__MODULE__{ + job_id: String.t() | nil, video_id: pos_integer(), transfer_id: String.t(), + filename: String.t() | nil, percent: number(), bytes_sent: non_neg_integer(), total_bytes: non_neg_integer(), + bytes_per_second: non_neg_integer() | nil, + eta: non_neg_integer() | nil, chunk_index: non_neg_integer() | nil, total_chunks: non_neg_integer() | nil } @@ -174,22 +182,42 @@ defmodule Reencodarr.AbAv1.WorkerProtocol do with {:ok, video_id} <- required_integer(payload, [:video_id, "video_id"], :invalid_transfer_progress), {:ok, transfer_id} <- - required_string(payload, [:transfer_id, "transfer_id"], :invalid_transfer_progress), + required_string( + payload, + [:transfer_id, "transfer_id", :job_id, "job_id"], + :invalid_transfer_progress + ), {:ok, percent} <- required_number(payload, [:percent, "percent"], :invalid_transfer_progress) do {:ok, %TransferProgress{ + job_id: optional_string(payload, [:job_id, "job_id"]), video_id: video_id, transfer_id: transfer_id, + filename: optional_string(payload, [:filename, "filename"]), percent: percent, bytes_sent: optional_integer(payload, [ :bytes_sent, "bytes_sent", + :received_bytes, + "received_bytes", :transferred_bytes, "transferred_bytes" ]) || 0, - total_bytes: optional_integer(payload, [:total_bytes, "total_bytes"]) || 0, + total_bytes: + optional_integer(payload, [ + :total_bytes, + "total_bytes", + :expected_bytes, + "expected_bytes" + ]) || 0, + bytes_per_second: + optional_integer(payload, [ + :bytes_per_second, + "bytes_per_second" + ]), + eta: optional_integer(payload, [:eta, "eta"]), chunk_index: optional_integer(payload, [:chunk_index, "chunk_index"]), total_chunks: optional_integer(payload, [:total_chunks, "total_chunks"]) }} diff --git a/lib/reencodarr_web/live/workers_live.ex b/lib/reencodarr_web/live/workers_live.ex index 927906f2..850ba157 100644 --- a/lib/reencodarr_web/live/workers_live.ex +++ b/lib/reencodarr_web/live/workers_live.ex @@ -196,12 +196,19 @@ defmodule ReencodarrWeb.WorkersLive do defp worker_transfer(%{transfer_progress: nil}), do: "Transfer -" defp worker_transfer(%{transfer_progress: progress}) do - [ + parts = [ {"Transfer", progress.percent, &"#{format_number(&1)}%"}, + {"Throughput", progress.bytes_per_second, &format_throughput/1}, + {"ETA", progress.eta, &format_eta/1}, + {"Bytes", progress.bytes_sent, &format_transfer_bytes(&1, progress.total_bytes)}, {"Chunk", progress.chunk_index, &to_string/1}, {"Total", progress.total_chunks, &to_string/1} ] - |> format_parts() + + case format_parts(parts) do + "-" -> "-" + details -> progress_filename(progress) <> details + end end defp worker_vmafs(%{active_video_id: nil}), do: "-" @@ -224,6 +231,9 @@ defmodule ReencodarrWeb.WorkersLive do "CRF #{Formatters.crf(vmaf.crf)} -> #{Formatters.vmaf_score(vmaf.score, 1)} (#{format_number(vmaf.percent)}%)" end + defp progress_filename(%{filename: nil}), do: "" + defp progress_filename(%{filename: filename}), do: "#{filename} - " + defp format_parts(parts) do parts |> Enum.reject(fn {_label, value, _formatter} -> is_nil(value) end) @@ -244,6 +254,28 @@ defmodule ReencodarrWeb.WorkersLive do defp format_number(nil), do: "-" + defp format_throughput(bytes_per_second) + when is_integer(bytes_per_second) and bytes_per_second >= 0 do + "#{Formatters.file_size(bytes_per_second)}/s" + end + + defp format_throughput(_), do: "-" + + defp format_eta(seconds) when is_integer(seconds) and seconds >= 0, do: "#{seconds}s" + defp format_eta(_), do: "-" + + defp format_transfer_bytes(bytes_sent, total_bytes) + when is_integer(bytes_sent) and bytes_sent >= 0 and is_integer(total_bytes) and + total_bytes > 0 do + "#{Formatters.file_size(bytes_sent)} / #{Formatters.file_size(total_bytes)}" + end + + defp format_transfer_bytes(bytes_sent, _total_bytes) + when is_integer(bytes_sent) and bytes_sent >= 0, + do: Formatters.file_size(bytes_sent) + + defp format_transfer_bytes(_bytes_sent, _total_bytes), do: "-" + defp status_badge_class(%{active_video_id: nil}), do: "inline-flex rounded-full border border-emerald-900 bg-emerald-950 px-2 py-1 text-xs font-medium text-emerald-300" diff --git a/test/reencodarr/ab_av1/worker_protocol_test.exs b/test/reencodarr/ab_av1/worker_protocol_test.exs index 43c8e865..193ffc6d 100644 --- a/test/reencodarr/ab_av1/worker_protocol_test.exs +++ b/test/reencodarr/ab_av1/worker_protocol_test.exs @@ -40,20 +40,27 @@ defmodule Reencodarr.AbAv1.WorkerProtocolTest do test "parses transfer progress into a typed payload" do assert {:ok, %TransferProgress{ + job_id: "job-1", video_id: 123, - transfer_id: "transfer-1", + transfer_id: "job-1", + filename: "movie.mkv", percent: 42.5, bytes_sent: 1_048_576, total_bytes: 2_097_152, + bytes_per_second: 524_288, + eta: 12, chunk_index: 3, total_chunks: 8 }} = WorkerProtocol.parse_transfer_progress(%{ "video_id" => 123, - "transfer_id" => "transfer-1", + "job_id" => "job-1", "percent" => 42.5, - "bytes_sent" => 1_048_576, - "total_bytes" => 2_097_152, + "filename" => "movie.mkv", + "received_bytes" => 1_048_576, + "expected_bytes" => 2_097_152, + "bytes_per_second" => 524_288, + "eta" => 12, "chunk_index" => 3, "total_chunks" => 8 }) diff --git a/test/reencodarr_web/channels/worker_channel_test.exs b/test/reencodarr_web/channels/worker_channel_test.exs index 0469f7e6..85abc77a 100644 --- a/test/reencodarr_web/channels/worker_channel_test.exs +++ b/test/reencodarr_web/channels/worker_channel_test.exs @@ -229,10 +229,13 @@ defmodule ReencodarrWeb.WorkerChannelTest do assert_reply push(socket, "transfer_progress", %{ "video_id" => video_id, - "transfer_id" => "transfer-1", + "job_id" => "job-1", + "filename" => Path.basename(video.path), "percent" => 25.5, - "bytes_sent" => 2_621_440, - "total_bytes" => 10_485_760, + "received_bytes" => 2_621_440, + "expected_bytes" => 10_485_760, + "bytes_per_second" => 1_048_576, + "eta" => 15, "chunk_index" => 2, "total_chunks" => 8 }), @@ -242,12 +245,20 @@ defmodule ReencodarrWeb.WorkerChannelTest do assert_receive {:transfer_progress, %{ video_id: ^video_id, - transfer_id: "transfer-1", + transfer_id: "job-1", + job_id: "job-1", + filename: filename, percent: 25.5, bytes_sent: 2_621_440, - total_bytes: 10_485_760 + total_bytes: 10_485_760, + bytes_per_second: 1_048_576, + eta: 15, + chunk_index: 2, + total_chunks: 8 }} + assert filename == Path.basename(video.path) + assert_reply push(socket, "crf_search_progress", %{ "video_id" => video_id, "percent" => 62.0, diff --git a/test/reencodarr_web/live/workers_live_test.exs b/test/reencodarr_web/live/workers_live_test.exs index 61e38dc7..cf556b0f 100644 --- a/test/reencodarr_web/live/workers_live_test.exs +++ b/test/reencodarr_web/live/workers_live_test.exs @@ -55,7 +55,14 @@ defmodule ReencodarrWeb.WorkersLiveTest do assert {:ok, _session} = WorkerSessions.set_transfer_progress("worker-server-1", %{ + job_id: "job-1", + transfer_id: "job-1", + filename: Path.basename(video.path), percent: 25.5, + bytes_sent: 2_621_440, + total_bytes: 10_485_760, + bytes_per_second: 1_048_576, + eta: 15, chunk_index: 2, total_chunks: 8 }) @@ -78,6 +85,9 @@ defmodule ReencodarrWeb.WorkersLiveTest do assert html =~ "FPS 12.5 fps" assert html =~ "ETA 90s" assert html =~ "Transfer 25.5%" + assert html =~ "Throughput 1.0 MiB/s" + assert html =~ "ETA 15s" + assert html =~ "Bytes 2.5 MiB / 10.0 MiB" assert html =~ "Chunk 2" assert html =~ "CRF 28.0 -> 95.4 (93.0%)" end From b8c0f0c947714bd2d19cf42353fd9cac05e458ef Mon Sep 17 00:00:00 2001 From: Mika Cohen Date: Tue, 7 Jul 2026 19:37:13 -0600 Subject: [PATCH 32/70] Show active worker transfer progress --- lib/reencodarr_web/live/workers_live.ex | 81 +++++++++++++++++++ .../reencodarr_web/live/workers_live_test.exs | 5 +- 2 files changed, 84 insertions(+), 2 deletions(-) diff --git a/lib/reencodarr_web/live/workers_live.ex b/lib/reencodarr_web/live/workers_live.ex index 850ba157..1acfb355 100644 --- a/lib/reencodarr_web/live/workers_live.ex +++ b/lib/reencodarr_web/live/workers_live.ex @@ -64,6 +64,54 @@ defmodule ReencodarrWeb.WorkersLive do
+ <%= if active_workers(@workers) != [] do %> +
+
+

Active Transfers

+

+ Live transfer progress while the worker is receiving the video. +

+
+ +
+ <%= for worker <- active_workers(@workers) do %> + <% progress = worker_transfer_progress(worker) %> +
+
+
+
+ {worker.client_worker_id} +
+
+ {worker_transfer_name(worker)} +
+
+ +
+ {format_number(progress.percent)}% +
+
+ +
+
+
+
+ +
+
Bytes {format_transfer_bytes(progress)}
+
Throughput {format_throughput(progress.bytes_per_second)}
+
ETA {format_eta(progress.eta)}
+
Chunk {format_chunk_progress(progress)}
+
+
+ <% end %> +
+
+ <% end %> +
@@ -141,6 +189,10 @@ defmodule ReencodarrWeb.WorkersLive do defp worker_status(%{active_video_id: nil}), do: "Idle" + defp worker_status(%{transfer_progress: _progress}), do: "transferring" + + defp worker_status(%{crf_search_progress: _progress}), do: "crf_searching" + defp worker_status(%{active_video_id: video_id}) do case Media.get_video(video_id) do %Media.Video{state: state} -> Atom.to_string(state) @@ -276,6 +328,35 @@ defmodule ReencodarrWeb.WorkersLive do defp format_transfer_bytes(_bytes_sent, _total_bytes), do: "-" + defp format_transfer_bytes(%{bytes_sent: bytes_sent, total_bytes: total_bytes}), + do: format_transfer_bytes(bytes_sent, total_bytes) + + defp active_workers(workers) do + Enum.filter(workers, fn %{transfer_progress: progress} -> not is_nil(progress) end) + end + + defp worker_transfer_progress(%{transfer_progress: progress}), do: progress + defp worker_transfer_progress(_worker), do: %{} + + defp worker_transfer_name(%{transfer_progress: %{filename: filename}}) + when is_binary(filename) and filename != "", + do: filename + + defp worker_transfer_name(%{active_video_id: video_id}) when is_integer(video_id), + do: "video ##{video_id}" + + defp worker_transfer_name(_worker), do: "transfer" + + defp format_chunk_progress(%{chunk_index: chunk_index, total_chunks: total_chunks}) + when is_integer(chunk_index) and is_integer(total_chunks) and total_chunks > 0 do + "#{chunk_index + 1} / #{total_chunks}" + end + + defp format_chunk_progress(%{chunk_index: chunk_index}) when is_integer(chunk_index), + do: Integer.to_string(chunk_index) + + defp format_chunk_progress(_), do: "-" + defp status_badge_class(%{active_video_id: nil}), do: "inline-flex rounded-full border border-emerald-900 bg-emerald-950 px-2 py-1 text-xs font-medium text-emerald-300" diff --git a/test/reencodarr_web/live/workers_live_test.exs b/test/reencodarr_web/live/workers_live_test.exs index cf556b0f..21192b56 100644 --- a/test/reencodarr_web/live/workers_live_test.exs +++ b/test/reencodarr_web/live/workers_live_test.exs @@ -76,7 +76,8 @@ defmodule ReencodarrWeb.WorkersLiveTest do send(view.pid, {:worker_sessions_updated, %{sessions: WorkerSessions.list()}}) html = render(view) - assert html =~ "analyzed" + assert html =~ "Active Transfers" + assert html =~ "transferring" assert html =~ "video ##{video.id}" assert html =~ "CPU 87.5%" assert html =~ "Mem 1.0 GiB / 4.0 GiB" @@ -88,7 +89,7 @@ defmodule ReencodarrWeb.WorkersLiveTest do assert html =~ "Throughput 1.0 MiB/s" assert html =~ "ETA 15s" assert html =~ "Bytes 2.5 MiB / 10.0 MiB" - assert html =~ "Chunk 2" + assert html =~ "Chunk 3 / 8" assert html =~ "CRF 28.0 -> 95.4 (93.0%)" end end From b4e8f1b6923c28acb69ed20aa695d8d7960d5269 Mon Sep 17 00:00:00 2001 From: Mika Cohen Date: Tue, 7 Jul 2026 20:11:52 -0600 Subject: [PATCH 33/70] Reuse CRF dashboard panel for workers --- lib/reencodarr/ab_av1/worker_sessions.ex | 2 +- lib/reencodarr_web/channels/worker_channel.ex | 13 + .../components/crf_search_components.ex | 394 ++++++++++++++++++ lib/reencodarr_web/live/dashboard_live.ex | 292 +------------ lib/reencodarr_web/live/workers_live.ex | 335 +++++++-------- .../channels/worker_channel_test.exs | 10 +- .../reencodarr_web/live/workers_live_test.exs | 53 ++- 7 files changed, 596 insertions(+), 503 deletions(-) create mode 100644 lib/reencodarr_web/components/crf_search_components.ex diff --git a/lib/reencodarr/ab_av1/worker_sessions.ex b/lib/reencodarr/ab_av1/worker_sessions.ex index 6545d867..c3f738d0 100644 --- a/lib/reencodarr/ab_av1/worker_sessions.ex +++ b/lib/reencodarr/ab_av1/worker_sessions.ex @@ -144,7 +144,7 @@ defmodule Reencodarr.AbAv1.WorkerSessions do def handle_call({:set_crf_search_progress, server_worker_id, progress}, _from, state) do update_session_reply(server_worker_id, state, fn session -> - %{session | crf_search_progress: progress} + %{session | transfer_progress: nil, crf_search_progress: progress} end) end diff --git a/lib/reencodarr_web/channels/worker_channel.ex b/lib/reencodarr_web/channels/worker_channel.ex index 90265718..6713d28d 100644 --- a/lib/reencodarr_web/channels/worker_channel.ex +++ b/lib/reencodarr_web/channels/worker_channel.ex @@ -11,9 +11,13 @@ defmodule ReencodarrWeb.WorkerChannel do alias Reencodarr.Media @crf_search_topic WorkerProtocol.crf_search_topic() + @worker_control_topic_prefix "worker_controls:" + + def worker_control_topic(worker_id), do: @worker_control_topic_prefix <> worker_id @impl true def join(@crf_search_topic, _payload, %{assigns: %{worker_id: worker_id}} = socket) do + Phoenix.PubSub.subscribe(Reencodarr.PubSub, worker_control_topic(worker_id)) {:ok, %{worker_id: worker_id}, socket} end @@ -95,6 +99,15 @@ defmodule ReencodarrWeb.WorkerChannel do end end + def handle_info({:worker_control, action}, socket) do + push(socket, "control", %{ + action: Atom.to_string(action), + video_id: socket.assigns[:current_video_id] + }) + + {:noreply, socket} + end + def handle_info(:stream_transfer_chunk, %{assigns: %{transfer_io_device: io_device}} = socket) when not is_nil(io_device) do read_transfer_chunk(socket, io_device) diff --git a/lib/reencodarr_web/components/crf_search_components.ex b/lib/reencodarr_web/components/crf_search_components.ex new file mode 100644 index 00000000..b1c45a83 --- /dev/null +++ b/lib/reencodarr_web/components/crf_search_components.ex @@ -0,0 +1,394 @@ +defmodule ReencodarrWeb.CrfSearchComponents do + @moduledoc "Shared CRF search display components." + + use Phoenix.Component + + alias Reencodarr.Formatters + alias ReencodarrWeb.ChartHelpers + + @service_status_styles %{ + running: "bg-green-100 text-green-800", + paused: "bg-yellow-100 text-yellow-800", + processing: "bg-blue-100 text-blue-800", + pausing: "bg-orange-100 text-orange-800", + idle: "bg-cyan-100 text-cyan-800", + checking: "bg-gray-100 text-gray-600 dashboard-soft-pulse", + stopped: "bg-red-100 text-red-800", + unknown: "bg-gray-100 text-gray-800" + } + + @service_status_labels %{ + running: "Running", + paused: "Paused", + processing: "Processing", + pausing: "Pausing", + idle: "Idle", + checking: "Checking...", + stopped: "Stopped", + unknown: "Unknown" + } + + attr :results, :list, required: true + attr :target_vmaf, :integer, required: true + attr :testing_crf, :float, default: nil + + def crf_search_chart(assigns) do + scores = Enum.map(assigns.results, & &1.score) + + vmaf_min = + min(assigns.target_vmaf - 3, Enum.min(scores, fn -> assigns.target_vmaf - 3 end) - 1) + + vmaf_max = + max(assigns.target_vmaf + 3, Enum.max(scores, fn -> assigns.target_vmaf + 3 end) + 1) + + {crf_min, crf_max} = ChartHelpers.crf_range_from_results(assigns.results) + + dots = + Enum.with_index(assigns.results, fn r, idx -> + %{ + x: ChartHelpers.crf_to_x(r.crf, crf_min, crf_max), + y: ChartHelpers.vmaf_to_y(r.score, vmaf_min, vmaf_max), + crf: r.crf, + score: r.score, + above: r.score >= assigns.target_vmaf, + is_latest: idx == length(assigns.results) - 1 + } + end) + + target_y = ChartHelpers.vmaf_to_y(assigns.target_vmaf, vmaf_min, vmaf_max) + + y_ticks = + for vmaf <- trunc(vmaf_min)..trunc(vmaf_max) do + %{value: vmaf, y: ChartHelpers.vmaf_to_y(vmaf, vmaf_min, vmaf_max)} + end + + x_ticks = + for crf <- ChartHelpers.generate_x_ticks(trunc(crf_min), trunc(crf_max)) do + %{value: crf, x: ChartHelpers.crf_to_x(crf, crf_min, crf_max)} + end + + assigns = + assign(assigns, + vmaf_min: vmaf_min, + vmaf_max: vmaf_max, + crf_min: crf_min, + crf_max: crf_max, + dots: dots, + target_y: target_y, + y_ticks: y_ticks, + x_ticks: x_ticks + ) + + ~H""" + + <%= for tick <- @y_ticks do %> + + <% end %> + + + + {@target_vmaf} + + + <%= for dot <- @dots do %> + + <%= if dot.is_latest do %> + + {Formatters.vmaf_score(dot.score, 1)} + + <% end %> + <% end %> + + <%= if @testing_crf do %> + + + CRF {Formatters.crf(@testing_crf)} + + <% end %> + + <%= for tick <- @y_ticks do %> + + {tick.value} + + <% end %> + + <%= for tick <- @x_ticks do %> + + {tick.value} + + <% end %> + + + + + """ + end + + attr :video, :map, required: true + attr :results, :list, required: true + attr :sample, :map, required: true + attr :progress, :any, required: true + attr :queue_count, :integer, default: 0 + attr :queue_items, :list, default: [] + attr :status, :atom, required: true + attr :show_controls, :boolean, default: true + attr :show_queue, :boolean, default: true + attr :suspend_event, :string, default: "suspend_crf_search" + attr :resume_event, :string, default: "resume_crf_search" + attr :fail_event, :string, default: "fail_crf_search" + attr :worker_id, :string, default: nil + + def crf_search_panel(assigns) do + ~H""" +
+
+

CRF Search

+ + {service_status_text(@status)} + +
+ + <%= if @video do %> +
+
+
{@video.filename}
+
+ {Formatters.file_size(@video.video_size)} + {@video.width}x{@video.height} + <%= if @video.hdr do %> + HDR + <% end %> + Target: {@video.target_vmaf} VMAF +
+
+ + <%= if @sample do %> +
+ Sample {@sample.sample_num}/{@sample.total_samples} - CRF {@sample.crf} +
+ <% end %> + + <.crf_progress progress={@progress} /> + + <.active_job_controls + :if={@show_controls} + status={@status} + suspend_event={@suspend_event} + resume_event={@resume_event} + fail_event={@fail_event} + worker_id={@worker_id} + /> + + <%= if length(@results) > 0 or @sample do %> +
+ <.crf_search_chart + results={@results} + target_vmaf={@video.target_vmaf} + testing_crf={@sample && @sample.crf} + /> + + <%= if length(@results) > 0 do %> +
+ <%= for result <- @results do %> +
+ + CRF {Formatters.crf(result.crf)} -> {Formatters.vmaf_score(result.score, 1)} VMAF + + {if result[:percent], do: "#{result.percent}%", else: "-"} +
+ <% end %> +
+ <% else %> +
+ Sampling CRF {Formatters.crf(@sample.crf)}... waiting for first completed VMAF result. +
+ <% end %> +
+ <% end %> +
+ <% else %> +
+ Queue: {@queue_count} + <%= if @status == :idle do %> + - Idle + <% end %> +
+ + <%= if @show_controls and @status == :paused do %> +
+ <.active_job_controls + status={@status} + suspend_event={@suspend_event} + resume_event={@resume_event} + fail_event={@fail_event} + worker_id={@worker_id} + /> +
+ <% end %> + <% end %> + + <%= if @show_queue and length(@queue_items) > 0 do %> +
+
Next up ({@queue_count}):
+ <%= for video <- Enum.take(@queue_items, 5) do %> +
+ +
{Path.basename(video.path)}
+
+ <% end %> +
+ <% end %> +
+ """ + end + + attr :progress, :any, required: true + + defp crf_progress(%{progress: progress} = assigns) when is_map(progress) do + assigns = + assign(assigns, + percent: Map.get(progress, :percent), + fps: Map.get(progress, :fps), + eta: Map.get(progress, :eta) + ) + + ~H""" +
+
+ Progress {format_number(@percent)}% + FPS {Formatters.fps(@fps)} + ETA {@eta}s +
+
+
+
+
+
+ """ + end + + defp crf_progress(assigns), do: ~H"" + + attr :status, :atom, required: true + attr :suspend_event, :string, required: true + attr :resume_event, :string, required: true + attr :fail_event, :string, required: true + attr :worker_id, :string, default: nil + + defp active_job_controls(assigns) do + ~H""" +
+ <%= if @status == :paused do %> + + <% else %> + + <% end %> + | + +
+ """ + end + + defp service_status_class(status), + do: @service_status_styles[status] || @service_status_styles.unknown + + defp service_status_text(status), + do: @service_status_labels[status] || @service_status_labels.unknown + + defp format_number(number) when is_integer(number), do: Integer.to_string(number) + + defp format_number(number) when is_float(number), + do: :erlang.float_to_binary(number, decimals: 1) + + defp progress_width(percent) when is_number(percent) do + percent + |> max(0) + |> min(100) + |> format_number() + |> Kernel.<>("%") + end + + defp progress_width(_percent), do: "0%" +end diff --git a/lib/reencodarr_web/live/dashboard_live.ex b/lib/reencodarr_web/live/dashboard_live.ex index 0b54ebe0..265e87c7 100644 --- a/lib/reencodarr_web/live/dashboard_live.ex +++ b/lib/reencodarr_web/live/dashboard_live.ex @@ -19,6 +19,7 @@ defmodule ReencodarrWeb.DashboardLive do alias Reencodarr.Media.VideoQueries import ReencodarrWeb.ChartComponents + import ReencodarrWeb.CrfSearchComponents require Logger @@ -377,297 +378,6 @@ defmodule ReencodarrWeb.DashboardLive do """ end - # CRF Search Scatter Plot (SVG visualization) - attr :results, :list, required: true - attr :target_vmaf, :integer, required: true - attr :testing_crf, :float, default: nil - - defp crf_search_chart(assigns) do - alias ReencodarrWeb.ChartHelpers - - # Calculate VMAF y-axis range from target ± results - scores = Enum.map(assigns.results, & &1.score) - - vmaf_min = - min(assigns.target_vmaf - 3, Enum.min(scores, fn -> assigns.target_vmaf - 3 end) - 1) - - vmaf_max = - max(assigns.target_vmaf + 3, Enum.max(scores, fn -> assigns.target_vmaf + 3 end) + 1) - - # Dynamic CRF axis range from actual results - {crf_min, crf_max} = ChartHelpers.crf_range_from_results(assigns.results) - - # Pre-compute dot positions - dots = - Enum.with_index(assigns.results, fn r, idx -> - %{ - x: ChartHelpers.crf_to_x(r.crf, crf_min, crf_max), - y: ChartHelpers.vmaf_to_y(r.score, vmaf_min, vmaf_max), - crf: r.crf, - score: r.score, - above: r.score >= assigns.target_vmaf, - is_latest: idx == length(assigns.results) - 1 - } - end) - - # Calculate target line y-coordinate - target_y = ChartHelpers.vmaf_to_y(assigns.target_vmaf, vmaf_min, vmaf_max) - - # Y-axis tick labels (every 1 VMAF) - y_ticks = - for vmaf <- trunc(vmaf_min)..trunc(vmaf_max) do - %{value: vmaf, y: ChartHelpers.vmaf_to_y(vmaf, vmaf_min, vmaf_max)} - end - - # X-axis tick labels (dynamic based on actual CRF range) - x_ticks = - for crf <- ChartHelpers.generate_x_ticks(trunc(crf_min), trunc(crf_max)) do - %{value: crf, x: ChartHelpers.crf_to_x(crf, crf_min, crf_max)} - end - - assigns = - assign(assigns, - vmaf_min: vmaf_min, - vmaf_max: vmaf_max, - crf_min: crf_min, - crf_max: crf_max, - dots: dots, - target_y: target_y, - y_ticks: y_ticks, - x_ticks: x_ticks - ) - - ~H""" - - - <%= for tick <- @y_ticks do %> - - <% end %> - - - - - {@target_vmaf} - - - - <%= for dot <- @dots do %> - - <%= if dot.is_latest do %> - - {Formatters.vmaf_score(dot.score, 1)} - - <% end %> - <% end %> - - - <%= if @testing_crf do %> - - - CRF {Formatters.crf(@testing_crf)} - - <% end %> - - - <%= for tick <- @y_ticks do %> - - {tick.value} - - <% end %> - - - <%= for tick <- @x_ticks do %> - - {tick.value} - - <% end %> - - - - - - """ - end - - # SVG coordinate helpers delegated to ChartHelpers (dynamic CRF range) - - # Row 2: CRF Search Panel Component - attr :video, :map, required: true - attr :results, :list, required: true - attr :sample, :map, required: true - attr :progress, :any, required: true - attr :queue_count, :integer, required: true - attr :queue_items, :list, required: true - attr :status, :atom, required: true - - defp crf_search_panel(assigns) do - ~H""" -
-
-

CRF Search

- - {service_status_text(@status)} - -
- - <%= if @video do %> - -
- -
-
{@video.filename}
-
- {Formatters.file_size(@video.video_size)} - {@video.width}x{@video.height} - <%= if @video.hdr do %> - HDR - <% end %> - Target: {@video.target_vmaf} VMAF -
-
- - - <%= if @sample do %> -
- Sample {@sample.sample_num}/{@sample.total_samples} — CRF {@sample.crf} -
- <% end %> - <.active_job_controls - status={@status} - suspend_event="suspend_crf_search" - resume_event="resume_crf_search" - fail_event="fail_crf_search" - /> - - - <%= if length(@results) > 0 or @sample do %> -
- <.crf_search_chart - results={@results} - target_vmaf={@video.target_vmaf} - testing_crf={@sample && @sample.crf} - /> - - - <%= if length(@results) > 0 do %> -
- <%= for result <- @results do %> -
- - CRF {Formatters.crf(result.crf)} → {Formatters.vmaf_score(result.score, 1)} VMAF - - {if result[:percent], do: "#{result.percent}%", else: "—"} -
- <% end %> -
- <% else %> -
- Sampling CRF {Formatters.crf(@sample.crf)}... waiting for first completed VMAF result. -
- <% end %> -
- <% end %> -
- <% else %> - -
- Queue: {@queue_count} - <%= if @status == :idle do %> - • Idle - <% end %> -
- - <%= if @status == :paused do %> -
- <.active_job_controls - status={@status} - suspend_event="suspend_crf_search" - resume_event="resume_crf_search" - fail_event="fail_crf_search" - /> -
- <% end %> - <% end %> - - - <%= if length(@queue_items) > 0 do %> -
-
Next up ({@queue_count}):
- <%= for video <- Enum.take(@queue_items, 5) do %> -
- -
{Path.basename(video.path)}
-
- <% end %> -
- <% end %> -
- """ - end - # Row 2: Encoding Panel Component attr :video, :map, required: true attr :vmaf, :map, required: true diff --git a/lib/reencodarr_web/live/workers_live.ex b/lib/reencodarr_web/live/workers_live.ex index 1acfb355..e68ddefc 100644 --- a/lib/reencodarr_web/live/workers_live.ex +++ b/lib/reencodarr_web/live/workers_live.ex @@ -9,6 +9,9 @@ defmodule ReencodarrWeb.WorkersLive do alias Reencodarr.Dashboard.Events alias Reencodarr.Formatters alias Reencodarr.Media + alias Reencodarr.Rules + + import ReencodarrWeb.CrfSearchComponents @refresh_interval 5_000 @@ -37,6 +40,19 @@ defmodule ReencodarrWeb.WorkersLive do def handle_info({_event, _data}, socket), do: {:noreply, socket} + @impl true + def handle_event("pause_worker_crf_search", %{"worker-id" => worker_id}, socket) do + control_worker(socket, worker_id, :pause, "Worker pause requested") + end + + def handle_event("resume_worker_crf_search", %{"worker-id" => worker_id}, socket) do + control_worker(socket, worker_id, :resume, "Worker resume requested") + end + + def handle_event("stop_worker_crf_search", %{"worker-id" => worker_id}, socket) do + control_worker(socket, worker_id, :stop, "Worker stop requested") + end + @impl true def render(assigns) do ~H""" @@ -64,134 +80,96 @@ defmodule ReencodarrWeb.WorkersLive do - <%= if active_workers(@workers) != [] do %> -
-
-

Active Transfers

-

- Live transfer progress while the worker is receiving the video. -

-
- -
- <%= for worker <- active_workers(@workers) do %> - <% progress = worker_transfer_progress(worker) %> -
-
-
-
- {worker.client_worker_id} -
-
- {worker_transfer_name(worker)} + <%= if @workers == [] do %> +
+ No workers connected. +
+ <% else %> +
+ <%= for worker <- @workers do %> +
+
+
+
+
+
{worker.client_worker_id}
+
+ server: {worker.server_worker_id} +
+ + {worker_status(worker)} +
-
- {format_number(progress.percent)}% -
+ <.crf_search_panel + video={worker_crf_video(worker)} + results={worker_crf_results(worker)} + sample={worker_crf_sample(worker)} + progress={worker.crf_search_progress} + status={worker_crf_status(worker)} + show_controls={true} + show_queue={false} + suspend_event="pause_worker_crf_search" + resume_event="resume_worker_crf_search" + fail_event="stop_worker_crf_search" + worker_id={worker.server_worker_id} + />
-
-
+
-
-
Bytes {format_transfer_bytes(progress)}
-
Throughput {format_throughput(progress.bytes_per_second)}
-
ETA {format_eta(progress.eta)}
-
Chunk {format_chunk_progress(progress)}
-
+ <%= if worker.transfer_progress do %> +
+

+ Input +

+
+
{format_transfer_bytes(worker.transfer_progress)}
+
{format_number(worker.transfer_progress.percent)}%
+
{format_throughput(worker.transfer_progress.bytes_per_second)}
+
ETA {format_eta(worker.transfer_progress.eta)}
+
Chunk {format_chunk_progress(worker.transfer_progress)}
+
+
+ <% end %> +
- <% end %> -
-
+
+ <% end %> + <% end %> - -
-
- - - - - - - - - - - - - - - - <%= if @workers == [] do %> - - - - <% end %> - - <%= for worker <- @workers do %> - - - - - - - - - - - - - <% end %> - -
WorkerStateVideoResourcesLive ProgressCRF/VMAF ResultsProtocolVersionConnectedLast Seen
- No workers connected. -
-
{worker.client_worker_id}
-
- server: {worker.server_worker_id} -
- 		- - {worker_status(worker)} - - - {worker_video(worker)} - -
CPU {worker_cpu(worker)}
-
Mem {worker_memory(worker)}
-
Disk {worker_disk(worker)}
- 		-
{worker_progress(worker)}
-
{worker_transfer(worker)}
- 		- {worker_vmafs(worker)} - - {worker.protocol_version} - - {worker.version} - - {DateTime.to_iso8601(worker.connected_at)} - - {DateTime.to_iso8601(worker.last_seen_at)} -
-
""" end + defp control_worker(socket, worker_id, action, message) do + Phoenix.PubSub.broadcast( + Reencodarr.PubSub, + ReencodarrWeb.WorkerChannel.worker_control_topic(worker_id), + {:worker_control, action} + ) + + {:noreply, put_flash(socket, :info, message)} + end + defp worker_status(%{active_video_id: nil}), do: "Idle" - defp worker_status(%{transfer_progress: _progress}), do: "transferring" + defp worker_status(%{crf_search_progress: progress}) when not is_nil(progress), do: "CRF search" - defp worker_status(%{crf_search_progress: _progress}), do: "crf_searching" + defp worker_status(%{transfer_progress: progress}) when not is_nil(progress), + do: "Receiving input" defp worker_status(%{active_video_id: video_id}) do case Media.get_video(video_id) do @@ -200,8 +178,59 @@ defmodule ReencodarrWeb.WorkersLive do end end - defp worker_video(%{active_video_id: nil}), do: "none" - defp worker_video(%{active_video_id: video_id}), do: "video ##{video_id}" + defp worker_crf_status(%{active_video_id: nil}), do: :idle + + defp worker_crf_status(%{crf_search_progress: progress}) when not is_nil(progress), + do: :processing + + defp worker_crf_status(%{active_video_id: _video_id}), do: :processing + + defp worker_crf_video(worker) do + case active_video(worker) do + %Media.Video{} = video -> + %{ + video_id: video.id, + filename: Path.basename(video.path), + video_size: video.size, + width: video.width, + height: video.height, + hdr: video.hdr, + target_vmaf: Rules.vmaf_target(video) + } + + nil -> + nil + end + end + + defp worker_crf_results(worker) do + case active_video_id(worker) do + nil -> + [] + + video_id -> + video_id + |> Media.get_vmafs_for_video() + |> Enum.sort_by(& &1.crf) + |> Enum.map(fn vmaf -> + %{crf: vmaf.crf, score: vmaf.score, percent: vmaf.percent} + end) + end + end + + defp worker_crf_sample(_worker), do: nil + + defp active_video(worker) do + case active_video_id(worker) do + nil -> nil + video_id -> Media.get_video(video_id) + end + end + + defp active_video_id(%{active_video_id: video_id}) when is_integer(video_id), do: video_id + defp active_video_id(%{crf_search_progress: %{video_id: video_id}}), do: video_id + defp active_video_id(%{transfer_progress: %{video_id: video_id}}), do: video_id + defp active_video_id(_worker), do: nil defp worker_cpu(%{resource_usage: %{cpu_percent: cpu_percent}}) when is_number(cpu_percent), do: "#{format_number(cpu_percent)}%" @@ -234,70 +263,6 @@ defmodule ReencodarrWeb.WorkersLive do defp worker_disk(_worker), do: "-" - defp worker_progress(%{crf_search_progress: nil}), do: "CRF -" - - defp worker_progress(%{crf_search_progress: progress}) do - [ - {"CRF", progress.percent, &"#{format_number(&1)}%"}, - {"FPS", progress.fps, &Formatters.fps/1}, - {"ETA", progress.eta, &"#{&1}s"} - ] - |> format_parts() - end - - defp worker_transfer(%{transfer_progress: nil}), do: "Transfer -" - - defp worker_transfer(%{transfer_progress: progress}) do - parts = [ - {"Transfer", progress.percent, &"#{format_number(&1)}%"}, - {"Throughput", progress.bytes_per_second, &format_throughput/1}, - {"ETA", progress.eta, &format_eta/1}, - {"Bytes", progress.bytes_sent, &format_transfer_bytes(&1, progress.total_bytes)}, - {"Chunk", progress.chunk_index, &to_string/1}, - {"Total", progress.total_chunks, &to_string/1} - ] - - case format_parts(parts) do - "-" -> "-" - details -> progress_filename(progress) <> details - end - end - - defp worker_vmafs(%{active_video_id: nil}), do: "-" - - defp worker_vmafs(%{active_video_id: video_id}) do - video_id - |> Media.get_vmafs_for_video() - |> Enum.sort_by(& &1.crf) - |> Enum.take(4) - |> case do - [] -> - "-" - - vmafs -> - Enum.map_join(vmafs, " / ", &format_vmaf/1) - end - end - - defp format_vmaf(vmaf) do - "CRF #{Formatters.crf(vmaf.crf)} -> #{Formatters.vmaf_score(vmaf.score, 1)} (#{format_number(vmaf.percent)}%)" - end - - defp progress_filename(%{filename: nil}), do: "" - defp progress_filename(%{filename: filename}), do: "#{filename} - " - - defp format_parts(parts) do - parts - |> Enum.reject(fn {_label, value, _formatter} -> is_nil(value) end) - |> Enum.map(fn {label, value, formatter} -> - "#{label} #{formatter.(value)}" - end) - |> case do - [] -> "-" - formatted -> Enum.join(formatted, " / ") - end - end - defp format_number(number) when is_integer(number), do: Integer.to_string(number) defp format_number(number) when is_float(number) do @@ -331,22 +296,6 @@ defmodule ReencodarrWeb.WorkersLive do defp format_transfer_bytes(%{bytes_sent: bytes_sent, total_bytes: total_bytes}), do: format_transfer_bytes(bytes_sent, total_bytes) - defp active_workers(workers) do - Enum.filter(workers, fn %{transfer_progress: progress} -> not is_nil(progress) end) - end - - defp worker_transfer_progress(%{transfer_progress: progress}), do: progress - defp worker_transfer_progress(_worker), do: %{} - - defp worker_transfer_name(%{transfer_progress: %{filename: filename}}) - when is_binary(filename) and filename != "", - do: filename - - defp worker_transfer_name(%{active_video_id: video_id}) when is_integer(video_id), - do: "video ##{video_id}" - - defp worker_transfer_name(_worker), do: "transfer" - defp format_chunk_progress(%{chunk_index: chunk_index, total_chunks: total_chunks}) when is_integer(chunk_index) and is_integer(total_chunks) and total_chunks > 0 do "#{chunk_index + 1} / #{total_chunks}" diff --git a/test/reencodarr_web/channels/worker_channel_test.exs b/test/reencodarr_web/channels/worker_channel_test.exs index 85abc77a..204a2924 100644 --- a/test/reencodarr_web/channels/worker_channel_test.exs +++ b/test/reencodarr_web/channels/worker_channel_test.exs @@ -280,8 +280,16 @@ defmodule ReencodarrWeb.WorkerChannelTest do assert filename == Path.basename(video.path) session = WorkerSessions.get(server_worker_id) - assert session.transfer_progress.percent == 25.5 + assert is_nil(session.transfer_progress) assert session.crf_search_progress.percent == 62.0 + + Phoenix.PubSub.broadcast( + Reencodarr.PubSub, + WorkerChannel.worker_control_topic(server_worker_id), + {:worker_control, :pause} + ) + + assert_push "control", %{action: "pause", video_id: ^video_id} after Application.delete_env(:reencodarr, :worker_token) end diff --git a/test/reencodarr_web/live/workers_live_test.exs b/test/reencodarr_web/live/workers_live_test.exs index 21192b56..4a67f73b 100644 --- a/test/reencodarr_web/live/workers_live_test.exs +++ b/test/reencodarr_web/live/workers_live_test.exs @@ -46,13 +46,6 @@ defmodule ReencodarrWeb.WorkersLiveTest do disk_total_bytes: 1_099_511_627_776 }) - assert {:ok, _session} = - WorkerSessions.set_crf_search_progress("worker-server-1", %{ - percent: 62.0, - fps: 12.5, - eta: 90 - }) - assert {:ok, _session} = WorkerSessions.set_transfer_progress("worker-server-1", %{ job_id: "job-1", @@ -67,6 +60,14 @@ defmodule ReencodarrWeb.WorkersLiveTest do total_chunks: 8 }) + assert {:ok, _session} = + WorkerSessions.set_crf_search_progress("worker-server-1", %{ + video_id: video.id, + percent: 62.0, + fps: 12.5, + eta: 90 + }) + Fixtures.vmaf_fixture(%{ video_id: video.id, crf: 28.0, @@ -76,20 +77,38 @@ defmodule ReencodarrWeb.WorkersLiveTest do send(view.pid, {:worker_sessions_updated, %{sessions: WorkerSessions.list()}}) html = render(view) - assert html =~ "Active Transfers" - assert html =~ "transferring" - assert html =~ "video ##{video.id}" + refute html =~ "Active Transfers" + refute html =~ "Protocol" + refute html =~ "Version" + assert html =~ "CRF Search" + assert html =~ "CRF search" + assert html =~ Path.basename(video.path) + assert html =~ "2.0 GiB" + assert html =~ "1920x1080" + assert html =~ "Target: 95 VMAF" assert html =~ "CPU 87.5%" assert html =~ "Mem 1.0 GiB / 4.0 GiB" assert html =~ "Disk 500.0 GiB free / 1.0 TiB" - assert html =~ "CRF 62.0%" + assert html =~ "Progress 62.0%" assert html =~ "FPS 12.5 fps" assert html =~ "ETA 90s" - assert html =~ "Transfer 25.5%" - assert html =~ "Throughput 1.0 MiB/s" - assert html =~ "ETA 15s" - assert html =~ "Bytes 2.5 MiB / 10.0 MiB" - assert html =~ "Chunk 3 / 8" - assert html =~ "CRF 28.0 -> 95.4 (93.0%)" + refute html =~ "Input" + refute html =~ "2.5 MiB / 10.0 MiB" + refute html =~ "Chunk 3 / 8" + assert html =~ "CRF 28.0 -> 95.4 VMAF" + assert html =~ "93.0%" + assert html =~ "Pause" + assert html =~ "Stop" + + Phoenix.PubSub.subscribe( + Reencodarr.PubSub, + ReencodarrWeb.WorkerChannel.worker_control_topic("worker-server-1") + ) + + view + |> element("button[phx-click='pause_worker_crf_search']") + |> render_click() + + assert_receive {:worker_control, :pause} end end From 77001bd9d81fcbb3b3dd9da5c261723ea84d7a52 Mon Sep 17 00:00:00 2001 From: Mika Cohen Date: Tue, 7 Jul 2026 20:41:43 -0600 Subject: [PATCH 34/70] Fix worker CRF resume state --- lib/reencodarr/ab_av1/worker_protocol.ex | 24 ++- lib/reencodarr/media.ex | 27 +++- lib/reencodarr/media/video.ex | 3 + lib/reencodarr/media/video_state_machine.ex | 6 + lib/reencodarr_web/channels/worker_channel.ex | 116 +++++++++---- .../components/crf_search_components.ex | 45 ------ lib/reencodarr_web/live/dashboard_live.ex | 1 - lib/reencodarr_web/live/workers_live.ex | 153 +++++++++++++----- ...000_add_crf_search_worker_id_to_videos.exs | 11 ++ .../ab_av1/worker_protocol_test.exs | 10 +- test/reencodarr/media/orphan_reset_test.exs | 17 ++ .../channels/worker_channel_test.exs | 89 +++++++++- .../reencodarr_web/live/workers_live_test.exs | 12 +- 13 files changed, 384 insertions(+), 130 deletions(-) create mode 100644 priv/repo/migrations/20260708000000_add_crf_search_worker_id_to_videos.exs diff --git a/lib/reencodarr/ab_av1/worker_protocol.ex b/lib/reencodarr/ab_av1/worker_protocol.ex index 0537f5e9..bad13379 100644 --- a/lib/reencodarr/ab_av1/worker_protocol.ex +++ b/lib/reencodarr/ab_av1/worker_protocol.ex @@ -63,14 +63,17 @@ defmodule Reencodarr.AbAv1.WorkerProtocol do @moduledoc false @enforce_keys [:video_id, :percent] - defstruct [:video_id, :percent, :filename, :eta, :fps] + defstruct [:video_id, :percent, :filename, :eta, :fps, :crf, :sample_num, :total_samples] @type t :: %__MODULE__{ video_id: pos_integer(), percent: number(), filename: String.t() | nil, eta: non_neg_integer() | nil, - fps: number() | nil + fps: number() | nil, + crf: number() | nil, + sample_num: pos_integer() | nil, + total_samples: pos_integer() | nil } end @@ -238,7 +241,10 @@ defmodule Reencodarr.AbAv1.WorkerProtocol do percent: percent, filename: optional_string(payload, [:filename, "filename"]), eta: optional_integer(payload, [:eta, "eta"]), - fps: optional_number(payload, [:fps, "fps"]) + fps: optional_number(payload, [:fps, "fps"]), + crf: optional_number(payload, [:crf, "crf"]), + sample_num: optional_integer(payload, [:sample_num, "sample_num"]), + total_samples: optional_integer(payload, [:total_samples, "total_samples"]) }} end end @@ -464,6 +470,18 @@ defmodule Reencodarr.AbAv1.WorkerProtocol do } end + def work_in_progress(%Video{id: video_id, path: path, size: size}, target_vmaf) + when is_integer(video_id) and is_binary(path) do + %{ + status: "job_in_progress", + job_id: Integer.to_string(video_id), + video_id: video_id, + source_name: Path.basename(path), + size_bytes: size || 0, + target_vmaf: target_vmaf + } + end + @spec heartbeat_ack(DateTime.t()) :: map() def heartbeat_ack(last_seen_at), do: %{accepted: true, last_seen_at: DateTime.to_iso8601(last_seen_at)} diff --git a/lib/reencodarr/media.ex b/lib/reencodarr/media.ex index bfeb3ab4..b755d0c5 100644 --- a/lib/reencodarr/media.ex +++ b/lib/reencodarr/media.ex @@ -312,6 +312,31 @@ defmodule Reencodarr.Media do def mark_as_crf_searching(%Video{} = video), do: VideoStateMachine.mark_as_crf_searching(video) + def mark_as_worker_crf_searching(%Video{id: video_id}, worker_id) + when is_integer(video_id) and is_binary(worker_id) do + write( + fn -> + from(v in Video, where: v.id == ^video_id and v.state == :crf_searching) + |> Repo.update_all(set: [crf_search_worker_id: worker_id, updated_at: DateTime.utc_now()]) + end, + label: :media_mark_as_worker_crf_searching + ) + |> case do + {1, _} -> {:ok, Repo.get(Video, video_id)} + _ -> {:error, :not_crf_searching} + end + end + + def get_worker_crf_searching_video(worker_id) when is_binary(worker_id) do + Repo.one( + from(v in Video, + where: v.state == :crf_searching and v.crf_search_worker_id == ^worker_id, + order_by: [desc: v.updated_at], + limit: 1 + ) + ) + end + def mark_as_encoding(%Video{} = video), do: VideoStateMachine.mark_as_encoding(video) def mark_as_reencoded(%Video{} = video), do: VideoStateMachine.mark_as_reencoded(video) @@ -1384,7 +1409,7 @@ defmodule Reencodarr.Media do def reset_orphaned_crf_searching do exclude_id = CrfSearch.current_video_id() - from(v in Video, where: v.state == :crf_searching) + from(v in Video, where: v.state == :crf_searching and is_nil(v.crf_search_worker_id)) |> maybe_exclude_video(exclude_id) |> reset_videos("orphaned crf_searching videos → analyzed") end diff --git a/lib/reencodarr/media/video.ex b/lib/reencodarr/media/video.ex index 964e332e..f806d3c9 100644 --- a/lib/reencodarr/media/video.ex +++ b/lib/reencodarr/media/video.ex @@ -40,6 +40,7 @@ defmodule Reencodarr.Media.Video do atmos: boolean() | nil, service_id: String.t() | nil, service_type: service_type() | nil, + crf_search_worker_id: String.t() | nil, mediainfo: map() | nil, inserted_at: DateTime.t() | nil, updated_at: DateTime.t() | nil @@ -78,6 +79,7 @@ defmodule Reencodarr.Media.Video do :library_id, :service_id, :service_type, + :crf_search_worker_id, :duration, :mediainfo, :chosen_vmaf_id, @@ -134,6 +136,7 @@ defmodule Reencodarr.Media.Video do field :title, :string field :service_id, :string field :service_type, Ecto.Enum, values: @service_types + field :crf_search_worker_id, :string field :mediainfo, :map # Year information from Sonarr/Radarr APIs diff --git a/lib/reencodarr/media/video_state_machine.ex b/lib/reencodarr/media/video_state_machine.ex index f5499dd3..32e74e3d 100644 --- a/lib/reencodarr/media/video_state_machine.ex +++ b/lib/reencodarr/media/video_state_machine.ex @@ -92,12 +92,18 @@ defmodule Reencodarr.Media.VideoStateMachine do video |> change(attrs) |> put_change(:state, to_state) + |> maybe_clear_crf_search_worker_id(to_state) |> validate_state_transition(from_state, to_state) {:ok, changeset} end end + defp maybe_clear_crf_search_worker_id(changeset, :crf_searching), do: changeset + + defp maybe_clear_crf_search_worker_id(changeset, _to_state), + do: put_change(changeset, :crf_search_worker_id, nil) + @doc """ Transitions a video to a new state with automatic state-specific validations. """ diff --git a/lib/reencodarr_web/channels/worker_channel.ex b/lib/reencodarr_web/channels/worker_channel.ex index 6713d28d..727b15ae 100644 --- a/lib/reencodarr_web/channels/worker_channel.ex +++ b/lib/reencodarr_web/channels/worker_channel.ex @@ -122,8 +122,11 @@ defmodule ReencodarrWeb.WorkerChannel do end @impl true - def terminate(_reason, %{assigns: %{worker_id: worker_id}} = socket) do - maybe_requeue_active_video(socket) + def terminate(reason, %{assigns: %{worker_id: worker_id}} = socket) do + if !shutdown_reason?(reason) do + maybe_requeue_active_video(socket) + end + WorkerSessions.unregister(worker_id) :ok end @@ -131,13 +134,16 @@ defmodule ReencodarrWeb.WorkerChannel do defp handle_work_request(_payload, %{assigns: %{worker_id: worker_id}} = socket) do case socket.assigns[:current_video_id] do nil -> - claim_work(worker_id, socket) + case resume_dispatched_work(socket) do + {:reply, reply, socket} -> {:reply, reply, socket} + :none -> claim_work(worker_id, socket) + end video_id -> case Media.get_video(video_id) do %Media.Video{} = video -> {:reply, - {:ok, WorkerProtocol.work_assigned(video, socket.assigns[:current_vmaf_target])}, + {:ok, WorkerProtocol.work_in_progress(video, socket.assigns[:current_vmaf_target])}, socket} nil -> @@ -146,6 +152,20 @@ defmodule ReencodarrWeb.WorkerChannel do end end + defp resume_dispatched_work(socket) do + case Media.get_worker_crf_searching_video(worker_dispatch_id(socket)) do + %Media.Video{} = video -> + with {:ok, socket} <- ensure_resumable_active_video(socket, video.id) do + {:reply, + {:ok, WorkerProtocol.work_in_progress(video, socket.assigns[:current_vmaf_target])}, + socket} + end + + nil -> + :none + end + end + defp handle_transfer_progress(payload, %{assigns: %{worker_id: worker_id}} = socket) do with {:ok, progress} <- WorkerProtocol.parse_transfer_progress(payload), :ok <- ensure_active_video(socket, progress.video_id) do @@ -160,7 +180,7 @@ defmodule ReencodarrWeb.WorkerChannel do defp handle_crf_search_progress(payload, socket) do with {:ok, progress} <- WorkerProtocol.parse_crf_search_progress(payload), - :ok <- ensure_active_video(socket, progress.video_id) do + {:ok, socket} <- ensure_resumable_active_video(socket, progress.video_id) do _ = WorkerSessions.set_crf_search_progress(socket.assigns.worker_id, progress) Events.broadcast_event(:crf_search_progress, progress) {:reply, {:ok, WorkerProtocol.event_ack("crf_search_progress")}, socket} @@ -172,7 +192,7 @@ defmodule ReencodarrWeb.WorkerChannel do defp handle_crf_search_result(payload, socket) do with {:ok, result} <- WorkerProtocol.parse_crf_search_result(payload), - :ok <- ensure_active_video(socket, result.video_id) do + {:ok, socket} <- ensure_resumable_active_video(socket, result.video_id) do handle_valid_crf_search_result(socket, result) else {:error, reason} -> @@ -182,7 +202,7 @@ defmodule ReencodarrWeb.WorkerChannel do defp handle_crf_search_completed(payload, %{assigns: %{worker_id: worker_id}} = socket) do with {:ok, completion} <- WorkerProtocol.parse_completion(payload), - :ok <- ensure_active_video(socket, completion.video_id) do + {:ok, socket} <- ensure_resumable_active_video(socket, completion.video_id) do handle_valid_crf_search_completion(worker_id, socket, completion) else {:error, reason} -> @@ -192,7 +212,7 @@ defmodule ReencodarrWeb.WorkerChannel do defp handle_video_failed(payload, %{assigns: %{worker_id: worker_id}} = socket) do with {:ok, failure} <- WorkerProtocol.parse_failure_report(payload), - :ok <- ensure_active_video(socket, failure.video_id) do + {:ok, socket} <- ensure_resumable_active_video(socket, failure.video_id) do video = Media.get_video(failure.video_id) if is_nil(video) do @@ -232,27 +252,27 @@ defmodule ReencodarrWeb.WorkerChannel do chunk_size_bytes = WorkerProtocol.chunk_size_bytes() total_chunks = total_chunks(total_bytes, chunk_size_bytes) - case WorkerSessions.assign_video(worker_id, video.id) do - {:ok, _session} -> - socket = - socket - |> assign(:current_video_id, video.id) - |> assign(:current_vmaf_target, target_vmaf) - |> assign(:transfer_io_device, nil) - |> assign(:transfer_path, video.path) - |> assign(:transfer_id, transfer_id) - |> assign(:transfer_chunk_size_bytes, chunk_size_bytes) - |> assign(:transfer_total_bytes, total_bytes) - |> assign(:transfer_total_chunks, total_chunks) - |> assign(:transfer_bytes_sent, 0) - |> assign(:transfer_chunk_index, 0) - - if File.exists?(video.path) do - send(self(), :stream_transfer_chunk) - end - - {:reply, {:ok, WorkerProtocol.work_assigned(video, target_vmaf)}, socket} + with {:ok, _video} <- Media.mark_as_worker_crf_searching(video, worker_dispatch_id(socket)), + {:ok, _session} <- WorkerSessions.assign_video(worker_id, video.id) do + socket = + socket + |> assign(:current_video_id, video.id) + |> assign(:current_vmaf_target, target_vmaf) + |> assign(:transfer_io_device, nil) + |> assign(:transfer_path, video.path) + |> assign(:transfer_id, transfer_id) + |> assign(:transfer_chunk_size_bytes, chunk_size_bytes) + |> assign(:transfer_total_bytes, total_bytes) + |> assign(:transfer_total_chunks, total_chunks) + |> assign(:transfer_bytes_sent, 0) + |> assign(:transfer_chunk_index, 0) + + if File.exists?(video.path) do + send(self(), :stream_transfer_chunk) + end + {:reply, {:ok, WorkerProtocol.work_assigned(video, target_vmaf)}, socket} + else {:error, reason} -> _ = Media.mark_as_analyzed(video) {:reply, {:error, WorkerProtocol.error(reason)}, socket} @@ -355,6 +375,46 @@ defmodule ReencodarrWeb.WorkerChannel do end end + defp ensure_resumable_active_video(socket, video_id) do + case socket.assigns[:current_video_id] do + ^video_id -> + {:ok, socket} + + nil -> + resume_active_video(socket, video_id) + + _other -> + {:error, :unknown_worker_session} + end + end + + defp resume_active_video(%{assigns: %{worker_id: worker_id}} = socket, video_id) do + with %Media.Video{state: :crf_searching, crf_search_worker_id: dispatch_id} = video + when is_binary(dispatch_id) <- Media.get_video(video_id), + true <- dispatch_id == worker_dispatch_id(socket), + false <- assigned_to_other_worker?(worker_id, video_id), + {:ok, _session} <- WorkerSessions.assign_video(worker_id, video_id) do + {:ok, + socket + |> assign(:current_video_id, video_id) + |> assign(:current_vmaf_target, Reencodarr.Rules.vmaf_target(video))} + else + _ -> {:error, :unknown_worker_session} + end + end + + defp assigned_to_other_worker?(worker_id, video_id) do + WorkerSessions.list() + |> Enum.any?(&(&1.server_worker_id != worker_id and &1.active_video_id == video_id)) + end + + defp shutdown_reason?(:shutdown), do: true + defp shutdown_reason?({:shutdown, _reason}), do: true + defp shutdown_reason?(_reason), do: false + + defp worker_dispatch_id(socket), + do: socket.assigns[:client_worker_id] || socket.assigns.worker_id + defp persist_crf_results(video, results) do Enum.each(results, fn result -> attrs = diff --git a/lib/reencodarr_web/components/crf_search_components.ex b/lib/reencodarr_web/components/crf_search_components.ex index b1c45a83..76620930 100644 --- a/lib/reencodarr_web/components/crf_search_components.ex +++ b/lib/reencodarr_web/components/crf_search_components.ex @@ -178,7 +178,6 @@ defmodule ReencodarrWeb.CrfSearchComponents do attr :video, :map, required: true attr :results, :list, required: true attr :sample, :map, required: true - attr :progress, :any, required: true attr :queue_count, :integer, default: 0 attr :queue_items, :list, default: [] attr :status, :atom, required: true @@ -219,8 +218,6 @@ defmodule ReencodarrWeb.CrfSearchComponents do <% end %> - <.crf_progress progress={@progress} /> - <.active_job_controls :if={@show_controls} status={@status} @@ -305,33 +302,6 @@ defmodule ReencodarrWeb.CrfSearchComponents do """ end - attr :progress, :any, required: true - - defp crf_progress(%{progress: progress} = assigns) when is_map(progress) do - assigns = - assign(assigns, - percent: Map.get(progress, :percent), - fps: Map.get(progress, :fps), - eta: Map.get(progress, :eta) - ) - - ~H""" -
-
- Progress {format_number(@percent)}% - FPS {Formatters.fps(@fps)} - ETA {@eta}s -
-
-
-
-
-
- """ - end - - defp crf_progress(assigns), do: ~H"" - attr :status, :atom, required: true attr :suspend_event, :string, required: true attr :resume_event, :string, required: true @@ -376,19 +346,4 @@ defmodule ReencodarrWeb.CrfSearchComponents do defp service_status_text(status), do: @service_status_labels[status] || @service_status_labels.unknown - - defp format_number(number) when is_integer(number), do: Integer.to_string(number) - - defp format_number(number) when is_float(number), - do: :erlang.float_to_binary(number, decimals: 1) - - defp progress_width(percent) when is_number(percent) do - percent - |> max(0) - |> min(100) - |> format_number() - |> Kernel.<>("%") - end - - defp progress_width(_percent), do: "0%" end diff --git a/lib/reencodarr_web/live/dashboard_live.ex b/lib/reencodarr_web/live/dashboard_live.ex index 265e87c7..2f1d6556 100644 --- a/lib/reencodarr_web/live/dashboard_live.ex +++ b/lib/reencodarr_web/live/dashboard_live.ex @@ -949,7 +949,6 @@ defmodule ReencodarrWeb.DashboardLive do video={@crf_search_video} results={@crf_search_results} sample={@crf_search_sample} - progress={@crf_progress} queue_count={@queue_counts.crf_searcher} queue_items={@queue_items.crf_searcher} status={@service_status.crf_searcher} diff --git a/lib/reencodarr_web/live/workers_live.ex b/lib/reencodarr_web/live/workers_live.ex index e68ddefc..46e3f313 100644 --- a/lib/reencodarr_web/live/workers_live.ex +++ b/lib/reencodarr_web/live/workers_live.ex @@ -87,26 +87,40 @@ defmodule ReencodarrWeb.WorkersLive do <% else %>
<%= for worker <- @workers do %> -
-
-
-
-
-
{worker.client_worker_id}
-
- server: {worker.server_worker_id} -
+
+ + +
+ <%= if show_crf_panel?(worker) do %> <.crf_search_panel video={worker_crf_video(worker)} results={worker_crf_results(worker)} sample={worker_crf_sample(worker)} - progress={worker.crf_search_progress} status={worker_crf_status(worker)} show_controls={true} show_queue={false} @@ -115,35 +129,9 @@ defmodule ReencodarrWeb.WorkersLive do fail_event="stop_worker_crf_search" worker_id={worker.server_worker_id} /> -
- - + <% else %> + <.transfer_panel worker={worker} video={active_video(worker)} /> + <% end %>
<% end %> @@ -164,6 +152,54 @@ defmodule ReencodarrWeb.WorkersLive do {:noreply, put_flash(socket, :info, message)} end + defp transfer_panel(assigns) do + ~H""" +
+
+

Receiving Input

+ + {format_number(@worker.transfer_progress && @worker.transfer_progress.percent)}% + +
+ + <%= if @video do %> +
+
{Path.basename(@video.path)}
+
+ {Formatters.file_size(@video.size)} + {format_dimensions(@video)} + <%= if @video.hdr do %> + HDR + <% end %> + Target: {Rules.vmaf_target(@video)} VMAF +
+
+ <% end %> + + <%= if @worker.transfer_progress do %> +
+
+
+
+
+ +
+ {format_transfer_bytes(@worker.transfer_progress)} + Chunk {format_chunk_progress(@worker.transfer_progress)} + {format_throughput(@worker.transfer_progress.bytes_per_second)} + ETA {format_eta(@worker.transfer_progress.eta)} +
+
+ <% else %> +
Waiting for work.
+ <% end %> +
+ """ + end + defp worker_status(%{active_video_id: nil}), do: "Idle" defp worker_status(%{crf_search_progress: progress}) when not is_nil(progress), do: "CRF search" @@ -185,6 +221,11 @@ defmodule ReencodarrWeb.WorkersLive do defp worker_crf_status(%{active_video_id: _video_id}), do: :processing + defp show_crf_panel?(%{crf_search_progress: progress}) when not is_nil(progress), do: true + defp show_crf_panel?(%{transfer_progress: progress}) when not is_nil(progress), do: false + defp show_crf_panel?(%{active_video_id: video_id}) when is_integer(video_id), do: true + defp show_crf_panel?(_worker), do: false + defp worker_crf_video(worker) do case active_video(worker) do %Media.Video{} = video -> @@ -218,6 +259,13 @@ defmodule ReencodarrWeb.WorkersLive do end end + defp worker_crf_sample(%{ + crf_search_progress: %{crf: crf, sample_num: sample_num, total_samples: total_samples} + }) + when is_number(crf) and is_integer(sample_num) and is_integer(total_samples) do + %{crf: crf, sample_num: sample_num, total_samples: total_samples} + end + defp worker_crf_sample(_worker), do: nil defp active_video(worker) do @@ -263,6 +311,11 @@ defmodule ReencodarrWeb.WorkersLive do defp worker_disk(_worker), do: "-" + defp worker_resource_usage?(%{resource_usage: usage}) when is_map(usage), + do: map_size(usage) > 0 + + defp worker_resource_usage?(_worker), do: false + defp format_number(number) when is_integer(number), do: Integer.to_string(number) defp format_number(number) when is_float(number) do @@ -271,6 +324,22 @@ defmodule ReencodarrWeb.WorkersLive do defp format_number(nil), do: "-" + defp progress_width(percent) when is_number(percent) do + percent + |> max(0) + |> min(100) + |> format_number() + |> Kernel.<>("%") + end + + defp progress_width(_percent), do: "0%" + + defp format_dimensions(%{width: width, height: height}) + when is_integer(width) and is_integer(height), + do: "#{width}x#{height}" + + defp format_dimensions(_video), do: "unknown resolution" + defp format_throughput(bytes_per_second) when is_integer(bytes_per_second) and bytes_per_second >= 0 do "#{Formatters.file_size(bytes_per_second)}/s" diff --git a/priv/repo/migrations/20260708000000_add_crf_search_worker_id_to_videos.exs b/priv/repo/migrations/20260708000000_add_crf_search_worker_id_to_videos.exs new file mode 100644 index 00000000..569eb9c0 --- /dev/null +++ b/priv/repo/migrations/20260708000000_add_crf_search_worker_id_to_videos.exs @@ -0,0 +1,11 @@ +defmodule Reencodarr.Repo.Migrations.AddCrfSearchWorkerIdToVideos do + use Ecto.Migration + + def change do + alter table(:videos) do + add :crf_search_worker_id, :string + end + + create index(:videos, [:state, :crf_search_worker_id]) + end +end diff --git a/test/reencodarr/ab_av1/worker_protocol_test.exs b/test/reencodarr/ab_av1/worker_protocol_test.exs index 193ffc6d..cfd6691a 100644 --- a/test/reencodarr/ab_av1/worker_protocol_test.exs +++ b/test/reencodarr/ab_av1/worker_protocol_test.exs @@ -73,14 +73,20 @@ defmodule Reencodarr.AbAv1.WorkerProtocolTest do percent: 78.5, filename: "movie.mkv", eta: 42, - fps: 23.97 + fps: 23.97, + crf: 24.0, + sample_num: 2, + total_samples: 8 }} = WorkerProtocol.parse_crf_search_progress(%{ "video_id" => 123, "percent" => 78.5, "filename" => "movie.mkv", "eta" => 42, - "fps" => 23.97 + "fps" => 23.97, + "crf" => 24.0, + "sample_num" => 2, + "total_samples" => 8 }) end diff --git a/test/reencodarr/media/orphan_reset_test.exs b/test/reencodarr/media/orphan_reset_test.exs index 001d42c2..211b7f35 100644 --- a/test/reencodarr/media/orphan_reset_test.exs +++ b/test/reencodarr/media/orphan_reset_test.exs @@ -22,6 +22,23 @@ defmodule Reencodarr.Media.OrphanResetTest do assert updated.state == :analyzed end + test "does not reset crf_searching videos dispatched to a worker" do + {:ok, video} = + Fixtures.video_fixture(%{ + path: "/test/worker_crf.mkv", + state: :crf_searching, + crf_search_worker_id: "worker-a", + video_codecs: ["h264"], + audio_codecs: ["aac"] + }) + + assert :ok = Media.reset_orphaned_crf_searching() + + updated = Media.get_video(video.id) + assert updated.state == :crf_searching + assert updated.crf_search_worker_id == "worker-a" + end + test "does not affect videos in other states" do {:ok, analyzed} = Fixtures.video_fixture(%{ diff --git a/test/reencodarr_web/channels/worker_channel_test.exs b/test/reencodarr_web/channels/worker_channel_test.exs index 204a2924..44a7003a 100644 --- a/test/reencodarr_web/channels/worker_channel_test.exs +++ b/test/reencodarr_web/channels/worker_channel_test.exs @@ -82,6 +82,7 @@ defmodule ReencodarrWeb.WorkerChannelTest do assert source_name == Path.basename(video.path) assert size_bytes == video.size assert Media.get_video(video.id).state == :crf_searching + assert Media.get_video(video.id).crf_search_worker_id == "worker-a" assert [session] = WorkerSessions.list() assert session.active_video_id == video.id after @@ -264,7 +265,10 @@ defmodule ReencodarrWeb.WorkerChannelTest do "percent" => 62.0, "filename" => Path.basename(video.path), "eta" => 90, - "fps" => 12.5 + "fps" => 12.5, + "crf" => 28.0, + "sample_num" => 3, + "total_samples" => 8 }), :ok, %{accepted: true, event: "crf_search_progress"} @@ -275,7 +279,10 @@ defmodule ReencodarrWeb.WorkerChannelTest do percent: 62.0, filename: filename, eta: 90, - fps: 12.5 + fps: 12.5, + crf: 28.0, + sample_num: 3, + total_samples: 8 }} assert filename == Path.basename(video.path) @@ -294,6 +301,78 @@ defmodule ReencodarrWeb.WorkerChannelTest do Application.delete_env(:reencodarr, :worker_token) end + test "accepts resumed CRF progress for already-searching video after reconnect" do + token = "test-worker-token" + Application.put_env(:reencodarr, :worker_token, token) + + {:ok, video} = + Fixtures.video_fixture(%{state: :crf_searching, crf_search_worker_id: "worker-a"}) + + video_id = video.id + + assert {:ok, socket} = connect(WorkerSocket, %{"token" => token}) + assert {:ok, _join_payload, socket} = subscribe_and_join(socket, "workers:crf_search") + server_worker_id = socket.assigns.worker_id + + assert_reply push(socket, "announce", announce_payload(worker_id: "worker-a")), + :ok, + %{accepted: true, protocol_version: 1} + + assert_reply push(socket, "crf_search_progress", %{ + "video_id" => video_id, + "percent" => 16.4, + "filename" => Path.basename(video.path), + "fps" => 24.0, + "crf" => 28.0, + "sample_num" => 2, + "total_samples" => 8 + }), + :ok, + %{accepted: true, event: "crf_search_progress"} + + session = WorkerSessions.get(server_worker_id) + assert session.active_video_id == video_id + assert session.crf_search_progress.sample_num == 2 + assert Media.get_video(video_id).state == :crf_searching + after + Application.delete_env(:reencodarr, :worker_token) + end + + test "does not restart transfer when reconnected worker asks for work already in progress" do + token = "test-worker-token" + Application.put_env(:reencodarr, :worker_token, token) + + {:ok, video} = + Fixtures.video_fixture(%{state: :crf_searching, crf_search_worker_id: "worker-a"}) + + video_id = video.id + + assert {:ok, socket} = connect(WorkerSocket, %{"token" => token}) + assert {:ok, _join_payload, socket} = subscribe_and_join(socket, "workers:crf_search") + server_worker_id = socket.assigns.worker_id + + assert_reply push(socket, "announce", announce_payload(worker_id: "worker-a")), + :ok, + %{accepted: true, protocol_version: 1} + + assert_reply push(socket, "pull_work", %{}), + :ok, + %{ + status: "job_in_progress", + video_id: ^video_id, + target_vmaf: 95 + } + + refute_push "transfer_started", _, 50 + refute_push "transfer_chunk", _, 50 + + session = WorkerSessions.get(server_worker_id) + assert session.active_video_id == video_id + assert Media.get_video(video_id).state == :crf_searching + after + Application.delete_env(:reencodarr, :worker_token) + end + test "streams assigned media in ordered chunks with integrity data" do token = "test-worker-token" previous_chunk_size = Application.get_env(:reencodarr, :worker_chunk_size_bytes) @@ -525,7 +604,7 @@ defmodule ReencodarrWeb.WorkerChannelTest do Application.delete_env(:reencodarr, :worker_token) end - test "requeues active work when the worker disconnects" do + test "keeps worker-dispatched active work when the worker disconnects" do token = "test-worker-token" Application.put_env(:reencodarr, :worker_token, token) @@ -548,7 +627,9 @@ defmodule ReencodarrWeb.WorkerChannelTest do Process.unlink(socket.channel_pid) assert :ok = close(socket) - assert Media.get_video(video.id).state == :analyzed + refreshed = Media.get_video(video.id) + assert refreshed.state == :crf_searching + assert refreshed.crf_search_worker_id == "worker-a" assert WorkerSessions.list() == [] after Application.delete_env(:reencodarr, :worker_token) diff --git a/test/reencodarr_web/live/workers_live_test.exs b/test/reencodarr_web/live/workers_live_test.exs index 4a67f73b..04fb2a53 100644 --- a/test/reencodarr_web/live/workers_live_test.exs +++ b/test/reencodarr_web/live/workers_live_test.exs @@ -65,7 +65,10 @@ defmodule ReencodarrWeb.WorkersLiveTest do video_id: video.id, percent: 62.0, fps: 12.5, - eta: 90 + eta: 90, + crf: 28.0, + sample_num: 3, + total_samples: 8 }) Fixtures.vmaf_fixture(%{ @@ -89,9 +92,10 @@ defmodule ReencodarrWeb.WorkersLiveTest do assert html =~ "CPU 87.5%" assert html =~ "Mem 1.0 GiB / 4.0 GiB" assert html =~ "Disk 500.0 GiB free / 1.0 TiB" - assert html =~ "Progress 62.0%" - assert html =~ "FPS 12.5 fps" - assert html =~ "ETA 90s" + assert html =~ "Sample 3/8 - CRF 28.0" + refute html =~ "Progress 62.0%" + refute html =~ "FPS 12.5 fps" + refute html =~ "ETA 90s" refute html =~ "Input" refute html =~ "2.5 MiB / 10.0 MiB" refute html =~ "Chunk 3 / 8" From 818d0882141b56ff1e78002843611a379a63fd2f Mon Sep 17 00:00:00 2001 From: Mika Cohen Date: Tue, 7 Jul 2026 21:50:09 -0600 Subject: [PATCH 35/70] Seed worker transfer progress on start --- lib/reencodarr/ab_av1/worker_sessions.ex | 4 +++- lib/reencodarr_web/channels/worker_channel.ex | 22 +++++++++++++++++++ .../channels/worker_channel_test.exs | 10 +++++++++ 3 files changed, 35 insertions(+), 1 deletion(-) diff --git a/lib/reencodarr/ab_av1/worker_sessions.ex b/lib/reencodarr/ab_av1/worker_sessions.ex index c3f738d0..14191a20 100644 --- a/lib/reencodarr/ab_av1/worker_sessions.ex +++ b/lib/reencodarr/ab_av1/worker_sessions.ex @@ -138,7 +138,9 @@ defmodule Reencodarr.AbAv1.WorkerSessions do def handle_call({:set_transfer_progress, server_worker_id, progress}, _from, state) do update_session_reply(server_worker_id, state, fn session -> - %{session | transfer_progress: progress} + active_video_id = Map.get(progress, :video_id, session.active_video_id) + + %{session | active_video_id: active_video_id, transfer_progress: progress} end) end diff --git a/lib/reencodarr_web/channels/worker_channel.ex b/lib/reencodarr_web/channels/worker_channel.ex index 727b15ae..e23db699 100644 --- a/lib/reencodarr_web/channels/worker_channel.ex +++ b/lib/reencodarr_web/channels/worker_channel.ex @@ -508,6 +508,12 @@ defmodule ReencodarrWeb.WorkerChannel do if is_nil(video) do {:noreply, handle_transfer_failure(socket, video_id, :enoent)} else + _ = + WorkerSessions.set_transfer_progress( + socket.assigns.worker_id, + initial_transfer_progress(socket, video) + ) + socket = socket |> assign(:transfer_started_sent, true) @@ -529,6 +535,22 @@ defmodule ReencodarrWeb.WorkerChannel do end end + defp initial_transfer_progress(socket, video) do + %WorkerProtocol.TransferProgress{ + job_id: socket.assigns.transfer_id, + video_id: video.id, + transfer_id: socket.assigns.transfer_id, + filename: Path.basename(video.path), + percent: 0.0, + bytes_sent: 0, + total_bytes: socket.assigns.transfer_total_bytes, + bytes_per_second: nil, + eta: nil, + chunk_index: 0, + total_chunks: socket.assigns.transfer_total_chunks + } + end + defp read_transfer_chunk(%{assigns: %{current_video_id: video_id}} = socket, io_device) do video = Media.get_video(video_id) diff --git a/test/reencodarr_web/channels/worker_channel_test.exs b/test/reencodarr_web/channels/worker_channel_test.exs index 44a7003a..1939a349 100644 --- a/test/reencodarr_web/channels/worker_channel_test.exs +++ b/test/reencodarr_web/channels/worker_channel_test.exs @@ -85,6 +85,7 @@ defmodule ReencodarrWeb.WorkerChannelTest do assert Media.get_video(video.id).crf_search_worker_id == "worker-a" assert [session] = WorkerSessions.list() assert session.active_video_id == video.id + assert is_nil(session.transfer_progress) after Application.delete_env(:reencodarr, :worker_token) end @@ -430,6 +431,15 @@ defmodule ReencodarrWeb.WorkerChannelTest do total_chunks: 2 } + session = WorkerSessions.get(socket.assigns.worker_id) + assert session.active_video_id == assigned_video_id + assert session.transfer_progress.video_id == assigned_video_id + assert session.transfer_progress.percent == 0.0 + assert session.transfer_progress.bytes_sent == 0 + assert session.transfer_progress.total_bytes == content_size + assert session.transfer_progress.chunk_index == 0 + assert session.transfer_progress.total_chunks == 2 + assert_push "transfer_chunk", {:binary, first_frame} assert {:ok, From cb0e2bc38e26eb163aec3c62e7c94abca3d48392 Mon Sep 17 00:00:00 2001 From: Mika Cohen Date: Tue, 7 Jul 2026 22:04:37 -0600 Subject: [PATCH 36/70] Stabilize worker reconnect state --- lib/reencodarr/ab_av1/worker_sessions.ex | 65 ++++++++++++++++- lib/reencodarr_web/channels/worker_channel.ex | 31 +++++++- .../components/crf_search_components.ex | 9 ++- lib/reencodarr_web/live/workers_live.ex | 1 + .../ab_av1/worker_sessions_test.exs | 73 +++++++++++++++++++ .../channels/worker_channel_test.exs | 17 +++-- 6 files changed, 183 insertions(+), 13 deletions(-) diff --git a/lib/reencodarr/ab_av1/worker_sessions.ex b/lib/reencodarr/ab_av1/worker_sessions.ex index 14191a20..d384279d 100644 --- a/lib/reencodarr/ab_av1/worker_sessions.ex +++ b/lib/reencodarr/ab_av1/worker_sessions.ex @@ -146,6 +146,8 @@ defmodule Reencodarr.AbAv1.WorkerSessions do def handle_call({:set_crf_search_progress, server_worker_id, progress}, _from, state) do update_session_reply(server_worker_id, state, fn session -> + progress = merge_crf_search_progress(session.crf_search_progress, progress) + %{session | transfer_progress: nil, crf_search_progress: progress} end) end @@ -240,6 +242,20 @@ defmodule Reencodarr.AbAv1.WorkerSessions do } end + defp merge_crf_search_progress( + %{video_id: video_id} = previous, + %{video_id: video_id} = progress + ) do + Enum.reduce([:crf, :sample_num, :total_samples], progress, fn key, merged -> + case Map.get(merged, key) do + nil -> Map.put(merged, key, Map.get(previous, key)) + _value -> merged + end + end) + end + + defp merge_crf_search_progress(_previous, progress), do: progress + defp register_session( server_worker_id, client_worker_id, @@ -266,8 +282,17 @@ defmodule Reencodarr.AbAv1.WorkerSessions do :ok = put_session(session) {:reply, {:ok, session}, state} - _existing_server_worker_id -> - {:reply, {:error, :duplicate_worker_id}, state} + existing_server_worker_id -> + replace_client_session( + existing_server_worker_id, + server_worker_id, + client_worker_id, + version, + protocol_version, + capabilities, + now, + state + ) end {:ok, %{client_worker_id: ^client_worker_id, connected_at: connected_at}} -> @@ -290,6 +315,42 @@ defmodule Reencodarr.AbAv1.WorkerSessions do end end + defp replace_client_session( + existing_server_worker_id, + server_worker_id, + client_worker_id, + version, + protocol_version, + capabilities, + now, + state + ) do + case lookup_session(existing_server_worker_id) do + {:ok, existing_session} -> + session = + build_session( + server_worker_id, + client_worker_id, + version, + protocol_version, + capabilities, + now + ) + |> Map.put(:connected_at, existing_session.connected_at) + |> Map.put(:active_video_id, existing_session.active_video_id) + |> Map.put(:transfer_progress, existing_session.transfer_progress) + |> Map.put(:crf_search_progress, existing_session.crf_search_progress) + |> Map.put(:resource_usage, existing_session.resource_usage) + + :ok = drop_session(existing_server_worker_id) + :ok = put_session(session) + {:reply, {:ok, session}, state} + + :error -> + {:reply, {:error, :duplicate_worker_id}, state} + end + end + defp required_attr(attrs, key), do: Map.fetch(attrs, key) defp update_session_reply(server_worker_id, state, update_fun) do diff --git a/lib/reencodarr_web/channels/worker_channel.ex b/lib/reencodarr_web/channels/worker_channel.ex index e23db699..0cc25a34 100644 --- a/lib/reencodarr_web/channels/worker_channel.ex +++ b/lib/reencodarr_web/channels/worker_channel.ex @@ -33,7 +33,7 @@ defmodule ReencodarrWeb.WorkerChannel do capabilities: capabilities }} <- WorkerProtocol.parse_announcement(payload), :ok <- ensure_supported_protocol_version(protocol_version), - {:ok, _session} <- + {:ok, session} <- WorkerSessions.register(%{ server_worker_id: server_worker_id, client_worker_id: client_worker_id, @@ -47,6 +47,7 @@ defmodule ReencodarrWeb.WorkerChannel do |> assign(:client_version, version) |> assign(:protocol_version, protocol_version) |> assign(:capabilities, capabilities) + |> attach_announced_work(client_worker_id, session) {:reply, {:ok, WorkerProtocol.accepted(protocol_version)}, socket} else @@ -152,6 +153,34 @@ defmodule ReencodarrWeb.WorkerChannel do end end + defp attach_announced_work(socket, _client_worker_id, %{active_video_id: video_id}) + when is_integer(video_id) do + attach_active_video(socket, video_id) + end + + defp attach_announced_work(socket, client_worker_id, _session) do + case Media.get_worker_crf_searching_video(client_worker_id) do + %Media.Video{} = video -> + _ = WorkerSessions.assign_video(socket.assigns.worker_id, video.id) + attach_active_video(socket, video.id) + + nil -> + socket + end + end + + defp attach_active_video(socket, video_id) do + case Media.get_video(video_id) do + %Media.Video{} = video -> + socket + |> assign(:current_video_id, video.id) + |> assign(:current_vmaf_target, Reencodarr.Rules.vmaf_target(video)) + + nil -> + socket + end + end + defp resume_dispatched_work(socket) do case Media.get_worker_crf_searching_video(worker_dispatch_id(socket)) do %Media.Video{} = video -> diff --git a/lib/reencodarr_web/components/crf_search_components.ex b/lib/reencodarr_web/components/crf_search_components.ex index 76620930..162b687e 100644 --- a/lib/reencodarr_web/components/crf_search_components.ex +++ b/lib/reencodarr_web/components/crf_search_components.ex @@ -183,6 +183,7 @@ defmodule ReencodarrWeb.CrfSearchComponents do attr :status, :atom, required: true attr :show_controls, :boolean, default: true attr :show_queue, :boolean, default: true + attr :show_empty_chart, :boolean, default: false attr :suspend_event, :string, default: "suspend_crf_search" attr :resume_event, :string, default: "resume_crf_search" attr :fail_event, :string, default: "fail_crf_search" @@ -227,7 +228,7 @@ defmodule ReencodarrWeb.CrfSearchComponents do worker_id={@worker_id} /> - <%= if length(@results) > 0 or @sample do %> + <%= if @show_empty_chart or length(@results) > 0 or @sample do %>
<.crf_search_chart results={@results} @@ -251,7 +252,11 @@ defmodule ReencodarrWeb.CrfSearchComponents do
<% else %>
- Sampling CRF {Formatters.crf(@sample.crf)}... waiting for first completed VMAF result. + <%= if @sample do %> + Sampling CRF {Formatters.crf(@sample.crf)}... waiting for first completed VMAF result. + <% else %> + Waiting for first CRF sample. + <% end %>
<% end %>
diff --git a/lib/reencodarr_web/live/workers_live.ex b/lib/reencodarr_web/live/workers_live.ex index 46e3f313..25e1b809 100644 --- a/lib/reencodarr_web/live/workers_live.ex +++ b/lib/reencodarr_web/live/workers_live.ex @@ -124,6 +124,7 @@ defmodule ReencodarrWeb.WorkersLive do status={worker_crf_status(worker)} show_controls={true} show_queue={false} + show_empty_chart={true} suspend_event="pause_worker_crf_search" resume_event="resume_worker_crf_search" fail_event="stop_worker_crf_search" diff --git a/test/reencodarr/ab_av1/worker_sessions_test.exs b/test/reencodarr/ab_av1/worker_sessions_test.exs index 5807feea..2e65bd1d 100644 --- a/test/reencodarr/ab_av1/worker_sessions_test.exs +++ b/test/reencodarr/ab_av1/worker_sessions_test.exs @@ -76,6 +76,79 @@ defmodule Reencodarr.AbAv1.WorkerSessionsTest do assert listed_session.active_video_id == 123 end + test "keeps CRF sample metadata when later progress omits it" do + assert {:ok, _session} = WorkerSessions.register(worker_session_attrs()) + + assert {:ok, session} = + WorkerSessions.set_crf_search_progress("worker-server-1", %{ + video_id: 123, + percent: 10.0, + fps: 24.0, + crf: 28.0, + sample_num: 3, + total_samples: 5 + }) + + assert session.crf_search_progress.crf == 28.0 + assert session.crf_search_progress.sample_num == 3 + assert session.crf_search_progress.total_samples == 5 + + assert {:ok, session} = + WorkerSessions.set_crf_search_progress("worker-server-1", %{ + video_id: 123, + percent: 25.0, + fps: 25.0, + crf: nil, + sample_num: nil, + total_samples: nil + }) + + assert session.crf_search_progress.percent == 25.0 + assert session.crf_search_progress.fps == 25.0 + assert session.crf_search_progress.crf == 28.0 + assert session.crf_search_progress.sample_num == 3 + assert session.crf_search_progress.total_samples == 5 + end + + test "replaces reconnecting client sessions without dropping active state" do + assert {:ok, _session} = + WorkerSessions.register( + worker_session_attrs( + server_worker_id: "worker-server-1", + client_worker_id: "worker-client-1" + ) + ) + + assert {:ok, _session} = WorkerSessions.assign_video("worker-server-1", 123) + + assert {:ok, session} = + WorkerSessions.set_crf_search_progress("worker-server-1", %{ + video_id: 123, + percent: 25.0, + fps: 24.0, + crf: 28.0, + sample_num: 2, + total_samples: 5 + }) + + assert session.active_video_id == 123 + + assert {:ok, session} = + WorkerSessions.register( + worker_session_attrs( + server_worker_id: "worker-server-2", + client_worker_id: "worker-client-1" + ) + ) + + assert session.server_worker_id == "worker-server-2" + assert session.client_worker_id == "worker-client-1" + assert session.active_video_id == 123 + assert session.crf_search_progress.video_id == 123 + assert session.crf_search_progress.sample_num == 2 + assert is_nil(WorkerSessions.get("worker-server-1")) + end + test "cancels and drains active distributed work" do {:ok, video_one} = Fixtures.video_fixture(%{state: :analyzed}) {:ok, video_two} = Fixtures.video_fixture(%{state: :analyzed}) diff --git a/test/reencodarr_web/channels/worker_channel_test.exs b/test/reencodarr_web/channels/worker_channel_test.exs index 1939a349..2359f85e 100644 --- a/test/reencodarr_web/channels/worker_channel_test.exs +++ b/test/reencodarr_web/channels/worker_channel_test.exs @@ -104,12 +104,13 @@ defmodule ReencodarrWeb.WorkerChannelTest do Application.delete_env(:reencodarr, :worker_token) end - test "rejects duplicate worker ids until the first session disconnects" do + test "replaces reconnecting worker sessions with the same client id" do token = "test-worker-token" Application.put_env(:reencodarr, :worker_token, token) assert {:ok, socket1} = connect(WorkerSocket, %{"token" => token}) assert {:ok, _join_payload, socket1} = subscribe_and_join(socket1, "workers:crf_search") + server_worker_id1 = socket1.assigns.worker_id assert_reply push(socket1, "announce", announce_payload()), :ok, @@ -117,17 +118,14 @@ defmodule ReencodarrWeb.WorkerChannelTest do assert {:ok, socket2} = connect(WorkerSocket, %{"token" => token}) assert {:ok, _join_payload, socket2} = subscribe_and_join(socket2, "workers:crf_search") - - assert_reply push(socket2, "announce", announce_payload()), - :error, - %{reason: "duplicate_worker_id"} - - Process.unlink(socket1.channel_pid) - assert :ok = close(socket1) + server_worker_id2 = socket2.assigns.worker_id assert_reply push(socket2, "announce", announce_payload()), :ok, %{accepted: true, protocol_version: 1} + + assert is_nil(WorkerSessions.get(server_worker_id1)) + assert WorkerSessions.get(server_worker_id2).client_worker_id == "abav1-dev" after Application.delete_env(:reencodarr, :worker_token) end @@ -319,6 +317,9 @@ defmodule ReencodarrWeb.WorkerChannelTest do :ok, %{accepted: true, protocol_version: 1} + session = WorkerSessions.get(server_worker_id) + assert session.active_video_id == video_id + assert_reply push(socket, "crf_search_progress", %{ "video_id" => video_id, "percent" => 16.4, From 03ebce2df64e8cd0f40095cb07c6e21c44d53f45 Mon Sep 17 00:00:00 2001 From: Mika Cohen Date: Tue, 7 Jul 2026 22:38:40 -0600 Subject: [PATCH 37/70] Send CRF rules to websocket workers --- lib/reencodarr/ab_av1/worker_protocol.ex | 11 ++-- lib/reencodarr_web/channels/worker_channel.ex | 35 +++++++++++-- .../channels/worker_channel_test.exs | 51 ++++++++++++++++++- 3 files changed, 87 insertions(+), 10 deletions(-) diff --git a/lib/reencodarr/ab_av1/worker_protocol.ex b/lib/reencodarr/ab_av1/worker_protocol.ex index bad13379..7aa191d2 100644 --- a/lib/reencodarr/ab_av1/worker_protocol.ex +++ b/lib/reencodarr/ab_av1/worker_protocol.ex @@ -3,6 +3,7 @@ defmodule Reencodarr.AbAv1.WorkerProtocol do Server-side helpers for the ab-av1 worker websocket protocol. """ + alias Reencodarr.AbAv1.CrfSearch alias Reencodarr.AbAv1.WorkerConfig alias Reencodarr.Media.Video @@ -457,7 +458,7 @@ defmodule Reencodarr.AbAv1.WorkerProtocol do def no_work, do: %{status: "no_work"} @spec work_assigned(Video.t(), number()) :: map() - def work_assigned(%Video{id: video_id, path: path, size: size}, target_vmaf) + def work_assigned(%Video{id: video_id, path: path, size: size} = video, target_vmaf) when is_integer(video_id) and is_binary(path) do %{ status: "job_assigned", @@ -466,11 +467,12 @@ defmodule Reencodarr.AbAv1.WorkerProtocol do source_name: Path.basename(path), size_bytes: size || 0, chunk_size_bytes: chunk_size_bytes(), - target_vmaf: target_vmaf + target_vmaf: target_vmaf, + crf_search_args: CrfSearch.build_crf_search_args(video, target_vmaf) } end - def work_in_progress(%Video{id: video_id, path: path, size: size}, target_vmaf) + def work_in_progress(%Video{id: video_id, path: path, size: size} = video, target_vmaf) when is_integer(video_id) and is_binary(path) do %{ status: "job_in_progress", @@ -478,7 +480,8 @@ defmodule Reencodarr.AbAv1.WorkerProtocol do video_id: video_id, source_name: Path.basename(path), size_bytes: size || 0, - target_vmaf: target_vmaf + target_vmaf: target_vmaf, + crf_search_args: CrfSearch.build_crf_search_args(video, target_vmaf) } end diff --git a/lib/reencodarr_web/channels/worker_channel.ex b/lib/reencodarr_web/channels/worker_channel.ex index 0cc25a34..c6a0d79b 100644 --- a/lib/reencodarr_web/channels/worker_channel.ex +++ b/lib/reencodarr_web/channels/worker_channel.ex @@ -135,10 +135,7 @@ defmodule ReencodarrWeb.WorkerChannel do defp handle_work_request(_payload, %{assigns: %{worker_id: worker_id}} = socket) do case socket.assigns[:current_video_id] do nil -> - case resume_dispatched_work(socket) do - {:reply, reply, socket} -> {:reply, reply, socket} - :none -> claim_work(worker_id, socket) - end + resume_or_claim_work(worker_id, socket) video_id -> case Media.get_video(video_id) do @@ -153,6 +150,36 @@ defmodule ReencodarrWeb.WorkerChannel do end end + defp resume_or_claim_work(worker_id, socket) do + case resume_session_work(worker_id, socket) do + {:reply, reply, socket} -> + {:reply, reply, socket} + + :none -> + case resume_dispatched_work(socket) do + {:reply, reply, socket} -> {:reply, reply, socket} + :none -> claim_work(worker_id, socket) + end + end + end + + defp resume_session_work(worker_id, socket) do + case WorkerSessions.get(worker_id) do + %{active_video_id: video_id} when is_integer(video_id) -> + with {:ok, socket} <- ensure_resumable_active_video(socket, video_id), + %Media.Video{} = video <- Media.get_video(video_id) do + {:reply, + {:ok, WorkerProtocol.work_in_progress(video, socket.assigns[:current_vmaf_target])}, + socket} + else + _ -> :none + end + + _session -> + :none + end + end + defp attach_announced_work(socket, _client_worker_id, %{active_video_id: video_id}) when is_integer(video_id) do attach_active_video(socket, video_id) diff --git a/test/reencodarr_web/channels/worker_channel_test.exs b/test/reencodarr_web/channels/worker_channel_test.exs index 2359f85e..fbc76624 100644 --- a/test/reencodarr_web/channels/worker_channel_test.exs +++ b/test/reencodarr_web/channels/worker_channel_test.exs @@ -74,13 +74,21 @@ defmodule ReencodarrWeb.WorkerChannelTest do video_id: assigned_video_id, source_name: source_name, size_bytes: size_bytes, - chunk_size_bytes: 134_217_728 + chunk_size_bytes: 134_217_728, + crf_search_args: crf_search_args } assert job_id == Integer.to_string(video.id) assert assigned_video_id == video.id assert source_name == Path.basename(video.path) assert size_bytes == video.size + assert "crf-search" in crf_search_args + assert "--input" in crf_search_args + assert video.path in crf_search_args + assert "--min-vmaf" in crf_search_args + assert "95" in crf_search_args + assert "--encoder" in crf_search_args + assert "svt-av1" in crf_search_args assert Media.get_video(video.id).state == :crf_searching assert Media.get_video(video.id).crf_search_worker_id == "worker-a" assert [session] = WorkerSessions.list() @@ -362,9 +370,16 @@ defmodule ReencodarrWeb.WorkerChannelTest do %{ status: "job_in_progress", video_id: ^video_id, - target_vmaf: 95 + target_vmaf: 95, + crf_search_args: crf_search_args } + assert "crf-search" in crf_search_args + assert "--input" in crf_search_args + assert video.path in crf_search_args + assert "--min-vmaf" in crf_search_args + assert "95" in crf_search_args + refute_push "transfer_started", _, 50 refute_push "transfer_chunk", _, 50 @@ -375,6 +390,38 @@ defmodule ReencodarrWeb.WorkerChannelTest do Application.delete_env(:reencodarr, :worker_token) end + test "returns in-progress work when an assigned worker asks again" do + token = "test-worker-token" + Application.put_env(:reencodarr, :worker_token, token) + + {:ok, video} = Fixtures.video_fixture(%{state: :analyzed}) + video_id = video.id + + assert {:ok, socket} = connect(WorkerSocket, %{"token" => token}) + assert {:ok, _join_payload, socket} = subscribe_and_join(socket, "workers:crf_search") + + assert_reply push(socket, "announce", announce_payload(worker_id: "worker-a")), + :ok, + %{accepted: true, protocol_version: 1} + + assert_reply push(socket, "pull_work", %{}), + :ok, + %{status: "job_assigned", video_id: ^video_id, crf_search_args: assigned_args} + + assert_reply push(socket, "pull_work", %{}), + :ok, + %{ + status: "job_in_progress", + video_id: ^video_id, + crf_search_args: in_progress_args + } + + assert assigned_args == in_progress_args + assert video.path in in_progress_args + after + Application.delete_env(:reencodarr, :worker_token) + end + test "streams assigned media in ordered chunks with integrity data" do token = "test-worker-token" previous_chunk_size = Application.get_env(:reencodarr, :worker_chunk_size_bytes) From c613cd1e4ccad8d1e8be4a26da9732377dd543b8 Mon Sep 17 00:00:00 2001 From: Mika Cohen Date: Tue, 7 Jul 2026 23:04:08 -0600 Subject: [PATCH 38/70] Resend worker input on request --- lib/reencodarr/ab_av1/crf_search.ex | 2 +- lib/reencodarr_web/channels/worker_channel.ex | 123 ++++++++++++------ .../ab_av1/worker_protocol_test.exs | 17 ++- .../channels/worker_channel_test.exs | 67 ++++++++++ 4 files changed, 168 insertions(+), 41 deletions(-) diff --git a/lib/reencodarr/ab_av1/crf_search.ex b/lib/reencodarr/ab_av1/crf_search.ex index bf903fcc..90058213 100644 --- a/lib/reencodarr/ab_av1/crf_search.ex +++ b/lib/reencodarr/ab_av1/crf_search.ex @@ -1174,7 +1174,7 @@ defmodule Reencodarr.AbAv1.CrfSearch do "--input", video.path, "--min-vmaf", - Integer.to_string(vmaf_percent), + to_string(vmaf_percent), "--min-crf", to_string(min_crf), "--max-crf", diff --git a/lib/reencodarr_web/channels/worker_channel.ex b/lib/reencodarr_web/channels/worker_channel.ex index c6a0d79b..b01ea25d 100644 --- a/lib/reencodarr_web/channels/worker_channel.ex +++ b/lib/reencodarr_web/channels/worker_channel.ex @@ -132,17 +132,17 @@ defmodule ReencodarrWeb.WorkerChannel do :ok end - defp handle_work_request(_payload, %{assigns: %{worker_id: worker_id}} = socket) do + defp handle_work_request(payload, %{assigns: %{worker_id: worker_id}} = socket) do + request_mode = work_request_mode(payload) + case socket.assigns[:current_video_id] do nil -> - resume_or_claim_work(worker_id, socket) + resume_or_claim_work(worker_id, socket, request_mode) video_id -> case Media.get_video(video_id) do %Media.Video{} = video -> - {:reply, - {:ok, WorkerProtocol.work_in_progress(video, socket.assigns[:current_vmaf_target])}, - socket} + reply_for_active_work(worker_id, socket, video, request_mode) nil -> {:reply, {:error, WorkerProtocol.error(:unknown_worker_session)}, socket} @@ -150,27 +150,25 @@ defmodule ReencodarrWeb.WorkerChannel do end end - defp resume_or_claim_work(worker_id, socket) do - case resume_session_work(worker_id, socket) do + defp resume_or_claim_work(worker_id, socket, request_mode) do + case resume_session_work(worker_id, socket, request_mode) do {:reply, reply, socket} -> {:reply, reply, socket} :none -> - case resume_dispatched_work(socket) do + case resume_dispatched_work(worker_id, socket, request_mode) do {:reply, reply, socket} -> {:reply, reply, socket} :none -> claim_work(worker_id, socket) end end end - defp resume_session_work(worker_id, socket) do + defp resume_session_work(worker_id, socket, request_mode) do case WorkerSessions.get(worker_id) do %{active_video_id: video_id} when is_integer(video_id) -> with {:ok, socket} <- ensure_resumable_active_video(socket, video_id), %Media.Video{} = video <- Media.get_video(video_id) do - {:reply, - {:ok, WorkerProtocol.work_in_progress(video, socket.assigns[:current_vmaf_target])}, - socket} + reply_for_active_work(worker_id, socket, video, request_mode) else _ -> :none end @@ -208,13 +206,11 @@ defmodule ReencodarrWeb.WorkerChannel do end end - defp resume_dispatched_work(socket) do + defp resume_dispatched_work(worker_id, socket, request_mode) do case Media.get_worker_crf_searching_video(worker_dispatch_id(socket)) do %Media.Video{} = video -> with {:ok, socket} <- ensure_resumable_active_video(socket, video.id) do - {:reply, - {:ok, WorkerProtocol.work_in_progress(video, socket.assigns[:current_vmaf_target])}, - socket} + reply_for_active_work(worker_id, socket, video, request_mode) end nil -> @@ -303,30 +299,10 @@ defmodule ReencodarrWeb.WorkerChannel do defp assign_claimed_work(worker_id, socket, video) do target_vmaf = Reencodarr.Rules.vmaf_target(video) - transfer_id = Integer.to_string(video.id) - total_bytes = video.size || 0 - chunk_size_bytes = WorkerProtocol.chunk_size_bytes() - total_chunks = total_chunks(total_bytes, chunk_size_bytes) with {:ok, _video} <- Media.mark_as_worker_crf_searching(video, worker_dispatch_id(socket)), {:ok, _session} <- WorkerSessions.assign_video(worker_id, video.id) do - socket = - socket - |> assign(:current_video_id, video.id) - |> assign(:current_vmaf_target, target_vmaf) - |> assign(:transfer_io_device, nil) - |> assign(:transfer_path, video.path) - |> assign(:transfer_id, transfer_id) - |> assign(:transfer_chunk_size_bytes, chunk_size_bytes) - |> assign(:transfer_total_bytes, total_bytes) - |> assign(:transfer_total_chunks, total_chunks) - |> assign(:transfer_bytes_sent, 0) - |> assign(:transfer_chunk_index, 0) - - if File.exists?(video.path) do - send(self(), :stream_transfer_chunk) - end - + socket = prepare_transfer(socket, video, target_vmaf) {:reply, {:ok, WorkerProtocol.work_assigned(video, target_vmaf)}, socket} else {:error, reason} -> @@ -335,6 +311,79 @@ defmodule ReencodarrWeb.WorkerChannel do end end + defp reply_for_active_work(worker_id, socket, video, :resend_input) do + target_vmaf = socket.assigns[:current_vmaf_target] || Reencodarr.Rules.vmaf_target(video) + + case WorkerSessions.assign_video(worker_id, video.id) do + {:ok, _session} -> + socket = prepare_transfer(socket, video, target_vmaf) + {:reply, {:ok, WorkerProtocol.work_assigned(video, target_vmaf)}, socket} + + {:error, reason} -> + {:reply, {:error, WorkerProtocol.error(reason)}, socket} + end + end + + defp reply_for_active_work(_worker_id, socket, video, :resume_only) do + {:reply, {:ok, WorkerProtocol.work_in_progress(video, socket.assigns[:current_vmaf_target])}, + socket} + end + + defp prepare_transfer(socket, video, target_vmaf) do + transfer_id = Integer.to_string(video.id) + total_bytes = video.size || 0 + chunk_size_bytes = WorkerProtocol.chunk_size_bytes() + total_chunks = total_chunks(total_bytes, chunk_size_bytes) + + socket = + socket + |> assign(:current_video_id, video.id) + |> assign(:current_vmaf_target, target_vmaf) + |> assign(:transfer_io_device, nil) + |> assign(:transfer_path, video.path) + |> assign(:transfer_id, transfer_id) + |> assign(:transfer_chunk_size_bytes, chunk_size_bytes) + |> assign(:transfer_total_bytes, total_bytes) + |> assign(:transfer_total_chunks, total_chunks) + |> assign(:transfer_bytes_sent, 0) + |> assign(:transfer_chunk_index, 0) + |> assign(:transfer_started_sent, nil) + + if File.exists?(video.path) do + send(self(), :stream_transfer_chunk) + end + + socket + end + + defp work_request_mode(payload) when is_map(payload) do + if Enum.any?( + [ + "input_missing", + :input_missing, + "needs_input", + :needs_input, + "request_transfer", + :request_transfer, + "resend_input", + :resend_input + ], + &truthy?(Map.get(payload, &1)) + ) do + :resend_input + else + :resume_only + end + end + + defp work_request_mode(_payload), do: :resume_only + + defp truthy?(true), do: true + defp truthy?("true"), do: true + defp truthy?("1"), do: true + defp truthy?(1), do: true + defp truthy?(_value), do: false + defp maybe_requeue_active_video(socket) do case socket.assigns[:current_video_id] do nil -> diff --git a/test/reencodarr/ab_av1/worker_protocol_test.exs b/test/reencodarr/ab_av1/worker_protocol_test.exs index cfd6691a..1f91e582 100644 --- a/test/reencodarr/ab_av1/worker_protocol_test.exs +++ b/test/reencodarr/ab_av1/worker_protocol_test.exs @@ -203,15 +203,26 @@ defmodule Reencodarr.AbAv1.WorkerProtocolTest do 96.5 ) - assert payload == %{ + assert %{ status: "job_assigned", job_id: "123", video_id: 123, source_name: "movie.mkv", size_bytes: 987_654, chunk_size_bytes: 134_217_728, - target_vmaf: 96.5 - } + target_vmaf: 96.5, + crf_search_args: crf_search_args + } = payload + + assert [ + "crf-search", + "--input", + "/videos/movie.mkv", + "--min-vmaf", + "96.5" | _rest + ] = crf_search_args + + assert "--temp-dir" in crf_search_args end test "builds binary transfer chunk frames with ordered metadata and raw data" do diff --git a/test/reencodarr_web/channels/worker_channel_test.exs b/test/reencodarr_web/channels/worker_channel_test.exs index fbc76624..4e2ff108 100644 --- a/test/reencodarr_web/channels/worker_channel_test.exs +++ b/test/reencodarr_web/channels/worker_channel_test.exs @@ -390,6 +390,73 @@ defmodule ReencodarrWeb.WorkerChannelTest do Application.delete_env(:reencodarr, :worker_token) end + test "resends active worker input when the worker reports it missing" do + token = "test-worker-token" + previous_chunk_size = Application.get_env(:reencodarr, :worker_chunk_size_bytes) + Application.put_env(:reencodarr, :worker_token, token) + Application.put_env(:reencodarr, :worker_chunk_size_bytes, 4) + + on_exit(fn -> + if is_nil(previous_chunk_size) do + Application.delete_env(:reencodarr, :worker_chunk_size_bytes) + else + Application.put_env(:reencodarr, :worker_chunk_size_bytes, previous_chunk_size) + end + end) + + content = "abcdefgh" + content_size = byte_size(content) + + with_temp_file(content, ".mkv", fn path -> + {:ok, video} = + Fixtures.video_fixture(%{ + path: path, + size: content_size, + state: :crf_searching, + crf_search_worker_id: "worker-a" + }) + + video_id = video.id + + assert {:ok, socket} = connect(WorkerSocket, %{"token" => token}) + assert {:ok, _join_payload, socket} = subscribe_and_join(socket, "workers:crf_search") + + assert_reply push(socket, "announce", announce_payload(worker_id: "worker-a")), + :ok, + %{accepted: true, protocol_version: 1} + + assert_reply push(socket, "pull_work", %{}), + :ok, + %{status: "job_in_progress", video_id: ^video_id} + + refute_push "transfer_started", _, 50 + + assert_reply push(socket, "pull_work", %{"input_missing" => true}), + :ok, + %{ + status: "job_assigned", + video_id: ^video_id, + chunk_size_bytes: 4 + } + + assert_push "transfer_started", %{ + status: "transfer_started", + video_id: ^video_id, + size_bytes: ^content_size, + chunk_size_bytes: 4, + total_bytes: ^content_size, + total_chunks: 2 + } + + assert_push "transfer_chunk", {:binary, first_frame} + + assert {:ok, %{video_id: ^video_id, chunk_index: 0, data: "abcd"}} = + WorkerProtocol.parse_transfer_chunk_frame(first_frame) + end) + after + Application.delete_env(:reencodarr, :worker_token) + end + test "returns in-progress work when an assigned worker asks again" do token = "test-worker-token" Application.put_env(:reencodarr, :worker_token, token) From 5a86b568d0434a276464f82c860031c7b6175c90 Mon Sep 17 00:00:00 2001 From: Mika Cohen Date: Tue, 7 Jul 2026 23:24:37 -0600 Subject: [PATCH 39/70] Resend worker input before progress --- lib/reencodarr_web/channels/worker_channel.ex | 17 +++++++++- .../channels/worker_channel_test.exs | 32 +++++++++++++++---- 2 files changed, 41 insertions(+), 8 deletions(-) diff --git a/lib/reencodarr_web/channels/worker_channel.ex b/lib/reencodarr_web/channels/worker_channel.ex index b01ea25d..3feab495 100644 --- a/lib/reencodarr_web/channels/worker_channel.ex +++ b/lib/reencodarr_web/channels/worker_channel.ex @@ -324,7 +324,15 @@ defmodule ReencodarrWeb.WorkerChannel do end end - defp reply_for_active_work(_worker_id, socket, video, :resume_only) do + defp reply_for_active_work(worker_id, socket, video, :resume_only) do + if initial_input_transfer_needed?(worker_id) do + reply_for_active_work(worker_id, socket, video, :resend_input) + else + resume_active_work(socket, video) + end + end + + defp resume_active_work(socket, video) do {:reply, {:ok, WorkerProtocol.work_in_progress(video, socket.assigns[:current_vmaf_target])}, socket} end @@ -378,6 +386,13 @@ defmodule ReencodarrWeb.WorkerChannel do defp work_request_mode(_payload), do: :resume_only + defp initial_input_transfer_needed?(worker_id) do + case WorkerSessions.get(worker_id) do + %{transfer_progress: nil, crf_search_progress: nil} -> true + _session -> false + end + end + defp truthy?(true), do: true defp truthy?("true"), do: true defp truthy?("1"), do: true diff --git a/test/reencodarr_web/channels/worker_channel_test.exs b/test/reencodarr_web/channels/worker_channel_test.exs index 4e2ff108..59e3e221 100644 --- a/test/reencodarr_web/channels/worker_channel_test.exs +++ b/test/reencodarr_web/channels/worker_channel_test.exs @@ -365,6 +365,18 @@ defmodule ReencodarrWeb.WorkerChannelTest do :ok, %{accepted: true, protocol_version: 1} + assert_reply push(socket, "crf_search_progress", %{ + "video_id" => video_id, + "percent" => 12.5, + "filename" => Path.basename(video.path), + "fps" => 24.0, + "crf" => 28.0, + "sample_num" => 1, + "total_samples" => 8 + }), + :ok, + %{accepted: true, event: "crf_search_progress"} + assert_reply push(socket, "pull_work", %{}), :ok, %{ @@ -390,7 +402,7 @@ defmodule ReencodarrWeb.WorkerChannelTest do Application.delete_env(:reencodarr, :worker_token) end - test "resends active worker input when the worker reports it missing" do + test "resends active worker input when active session has no progress yet" do token = "test-worker-token" previous_chunk_size = Application.get_env(:reencodarr, :worker_chunk_size_bytes) Application.put_env(:reencodarr, :worker_token, token) @@ -426,12 +438,6 @@ defmodule ReencodarrWeb.WorkerChannelTest do %{accepted: true, protocol_version: 1} assert_reply push(socket, "pull_work", %{}), - :ok, - %{status: "job_in_progress", video_id: ^video_id} - - refute_push "transfer_started", _, 50 - - assert_reply push(socket, "pull_work", %{"input_missing" => true}), :ok, %{ status: "job_assigned", @@ -475,6 +481,18 @@ defmodule ReencodarrWeb.WorkerChannelTest do :ok, %{status: "job_assigned", video_id: ^video_id, crf_search_args: assigned_args} + assert_reply push(socket, "crf_search_progress", %{ + "video_id" => video_id, + "percent" => 12.5, + "filename" => Path.basename(video.path), + "fps" => 24.0, + "crf" => 28.0, + "sample_num" => 1, + "total_samples" => 8 + }), + :ok, + %{accepted: true, event: "crf_search_progress"} + assert_reply push(socket, "pull_work", %{}), :ok, %{ From f728171bfd8a814d9cd1e979759e92c84e9160d9 Mon Sep 17 00:00:00 2001 From: Mika Cohen Date: Tue, 7 Jul 2026 23:44:06 -0600 Subject: [PATCH 40/70] Accept worker CRF result payloads --- lib/reencodarr/ab_av1/worker_protocol.ex | 24 ++++++++++++++--- .../ab_av1/worker_protocol_test.exs | 26 +++++++++++++++++++ .../channels/worker_channel_test.exs | 16 ++++++++++++ 3 files changed, 62 insertions(+), 4 deletions(-) diff --git a/lib/reencodarr/ab_av1/worker_protocol.ex b/lib/reencodarr/ab_av1/worker_protocol.ex index 7aa191d2..8aa0bfe5 100644 --- a/lib/reencodarr/ab_av1/worker_protocol.ex +++ b/lib/reencodarr/ab_av1/worker_protocol.ex @@ -578,7 +578,14 @@ defmodule Reencodarr.AbAv1.WorkerProtocol do {:ok, percent} <- required_number( payload, - [:percent, "percent", :vmaf_percentile, "vmaf_percentile"], + [ + :percent, + "percent", + :vmaf_percentile, + "vmaf_percentile", + :encode_percent, + "encode_percent" + ], :invalid_crf_search_result ) do {:ok, @@ -785,7 +792,12 @@ defmodule Reencodarr.AbAv1.WorkerProtocol do end defp optional_predicted_size(payload) do - case optional_number(payload, [:predicted_size, "predicted_size"]) do + case optional_number(payload, [ + :predicted_size, + "predicted_size", + :predicted_encode_size, + "predicted_encode_size" + ]) do nil -> nil size -> format_size(size, payload) end @@ -805,8 +817,12 @@ defmodule Reencodarr.AbAv1.WorkerProtocol do round(value) _ -> - case {optional_number(payload, [:time_taken, "time_taken"]), - optional_string(payload, [:time_unit, "time_unit"])} do + case {optional_number(payload, [ + :time_taken, + "time_taken", + :predicted_encode_time_secs, + "predicted_encode_time_secs" + ]), optional_string(payload, [:time_unit, "time_unit"])} do {nil, _} -> nil {time_taken, unit} when unit in [nil, ""] -> round(time_taken) {time_taken, unit} -> round(Reencodarr.Core.Time.to_seconds(time_taken, unit)) diff --git a/test/reencodarr/ab_av1/worker_protocol_test.exs b/test/reencodarr/ab_av1/worker_protocol_test.exs index 1f91e582..14730669 100644 --- a/test/reencodarr/ab_av1/worker_protocol_test.exs +++ b/test/reencodarr/ab_av1/worker_protocol_test.exs @@ -155,6 +155,32 @@ defmodule Reencodarr.AbAv1.WorkerProtocolTest do "chosen" => false }) + assert {:ok, + %CrfSearchResult{ + video_id: 123, + results: [ + %{ + crf: 31.5, + score: 96.2, + percent: 42.5, + size: "123456", + time: 88 + } + ] + }} = + WorkerProtocol.parse_crf_search_result(%{ + "job_id" => "job-123", + "video_id" => 123, + "source_name" => "movie.mkv", + "crf" => 31.5, + "vmaf_score" => 96.2, + "xpsnr_score" => nil, + "predicted_encode_size" => 123_456, + "encode_percent" => 42.5, + "predicted_encode_time_secs" => 87.5, + "from_cache" => false + }) + assert {:ok, %FailureReport{ retriable: false diff --git a/test/reencodarr_web/channels/worker_channel_test.exs b/test/reencodarr_web/channels/worker_channel_test.exs index 59e3e221..1250eb9e 100644 --- a/test/reencodarr_web/channels/worker_channel_test.exs +++ b/test/reencodarr_web/channels/worker_channel_test.exs @@ -658,6 +658,22 @@ defmodule ReencodarrWeb.WorkerChannelTest do assert_receive {:crf_search_vmaf_result, %{video_id: ^video_id, crf: 26.0, score: 94.1}} assert_receive {:crf_search_vmaf_result, %{video_id: ^video_id, crf: 28.0, score: 96.4}} + assert_reply push(socket, "crf_search_result", %{ + "job_id" => Integer.to_string(video_id), + "video_id" => video_id, + "source_name" => Path.basename(video.path), + "crf" => 30.0, + "vmaf_score" => 95.8, + "predicted_encode_size" => 123_456, + "encode_percent" => 42.5, + "predicted_encode_time_secs" => 87.5, + "from_cache" => false + }), + :ok, + %{accepted: true, event: "crf_search_result"} + + assert_receive {:crf_search_vmaf_result, %{video_id: ^video_id, crf: 30.0, score: 95.8}} + assert_reply push(socket, "crf_search_completed", %{ "video_id" => video_id, "result" => "ok", From 344641156c35365f7e6513d08977e05e9dd60efc Mon Sep 17 00:00:00 2001 From: Mika Cohen Date: Tue, 7 Jul 2026 23:50:52 -0600 Subject: [PATCH 41/70] Persist worker CRF results --- lib/reencodarr_web/channels/worker_channel.ex | 11 ++++++++++- test/reencodarr_web/channels/worker_channel_test.exs | 9 +++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/lib/reencodarr_web/channels/worker_channel.ex b/lib/reencodarr_web/channels/worker_channel.ex index 3feab495..4128e250 100644 --- a/lib/reencodarr_web/channels/worker_channel.ex +++ b/lib/reencodarr_web/channels/worker_channel.ex @@ -5,7 +5,7 @@ defmodule ReencodarrWeb.WorkerChannel do use ReencodarrWeb, :channel - alias Reencodarr.AbAv1.{WorkerProtocol, WorkerSessions} + alias Reencodarr.AbAv1.{CrfSearch, WorkerProtocol, WorkerSessions} alias Reencodarr.AbAv1.WorkerProtocol.Announcement alias Reencodarr.Dashboard.Events alias Reencodarr.Media @@ -537,9 +537,15 @@ defmodule ReencodarrWeb.WorkerChannel do defp persist_crf_results(video, results) do Enum.each(results, fn result -> + params = CrfSearch.build_crf_search_args(video, result_target(video, result)) + attrs = result |> Map.put(:video_id, video.id) + |> Map.update(:params, params, fn + nil -> params + existing_params -> existing_params + end) |> Map.delete(:chosen) case Media.upsert_vmaf(attrs) do @@ -550,6 +556,9 @@ defmodule ReencodarrWeb.WorkerChannel do end) end + defp result_target(video, result), + do: Map.get(result, :target) || Reencodarr.Rules.vmaf_target(video) + defp choose_result_from_report(video, results) do chosen_crf = Enum.find_value(results, fn result -> diff --git a/test/reencodarr_web/channels/worker_channel_test.exs b/test/reencodarr_web/channels/worker_channel_test.exs index 1250eb9e..144466ef 100644 --- a/test/reencodarr_web/channels/worker_channel_test.exs +++ b/test/reencodarr_web/channels/worker_channel_test.exs @@ -674,6 +674,15 @@ defmodule ReencodarrWeb.WorkerChannelTest do assert_receive {:crf_search_vmaf_result, %{video_id: ^video_id, crf: 30.0, score: 95.8}} + vmafs = Media.get_vmafs_for_video(video_id) + assert Enum.any?(vmafs, &(&1.crf == 30.0 and &1.score == 95.8)) + + worker_vmaf = Enum.find(vmafs, &(&1.crf == 30.0)) + assert worker_vmaf.percent == 42.5 + assert worker_vmaf.size == "123456" + assert worker_vmaf.time == 88 + assert is_list(worker_vmaf.params) + assert_reply push(socket, "crf_search_completed", %{ "video_id" => video_id, "result" => "ok", From b776e3b382d6da4eb7c452cfc5ad6cf9e1dae1fc Mon Sep 17 00:00:00 2001 From: Mika Cohen Date: Wed, 8 Jul 2026 00:33:06 -0600 Subject: [PATCH 42/70] Accept dispatched worker results after reconnect --- lib/reencodarr_web/channels/worker_channel.ex | 28 ++++++++++++- .../channels/worker_channel_test.exs | 42 +++++++++++++++++++ 2 files changed, 68 insertions(+), 2 deletions(-) diff --git a/lib/reencodarr_web/channels/worker_channel.ex b/lib/reencodarr_web/channels/worker_channel.ex index 4128e250..7dd7d776 100644 --- a/lib/reencodarr_web/channels/worker_channel.ex +++ b/lib/reencodarr_web/channels/worker_channel.ex @@ -244,7 +244,7 @@ defmodule ReencodarrWeb.WorkerChannel do defp handle_crf_search_result(payload, socket) do with {:ok, result} <- WorkerProtocol.parse_crf_search_result(payload), - {:ok, socket} <- ensure_resumable_active_video(socket, result.video_id) do + {:ok, socket} <- ensure_result_video(socket, result.video_id) do handle_valid_crf_search_result(socket, result) else {:error, reason} -> @@ -254,7 +254,7 @@ defmodule ReencodarrWeb.WorkerChannel do defp handle_crf_search_completed(payload, %{assigns: %{worker_id: worker_id}} = socket) do with {:ok, completion} <- WorkerProtocol.parse_completion(payload), - {:ok, socket} <- ensure_resumable_active_video(socket, completion.video_id) do + {:ok, socket} <- ensure_result_video(socket, completion.video_id) do handle_valid_crf_search_completion(worker_id, socket, completion) else {:error, reason} -> @@ -508,6 +508,30 @@ defmodule ReencodarrWeb.WorkerChannel do end end + defp ensure_result_video(socket, video_id) do + case ensure_resumable_active_video(socket, video_id) do + {:ok, socket} -> + {:ok, socket} + + {:error, :unknown_worker_session} -> + allow_dispatched_video_result(socket, video_id) + end + end + + defp allow_dispatched_video_result(socket, video_id) do + case Media.get_video(video_id) do + %Media.Video{state: :crf_searching, crf_search_worker_id: dispatch_id} = video + when is_binary(dispatch_id) -> + {:ok, + socket + |> assign(:current_video_id, video.id) + |> assign(:current_vmaf_target, Reencodarr.Rules.vmaf_target(video))} + + _ -> + {:error, :unknown_worker_session} + end + end + defp resume_active_video(%{assigns: %{worker_id: worker_id}} = socket, video_id) do with %Media.Video{state: :crf_searching, crf_search_worker_id: dispatch_id} = video when is_binary(dispatch_id) <- Media.get_video(video_id), diff --git a/test/reencodarr_web/channels/worker_channel_test.exs b/test/reencodarr_web/channels/worker_channel_test.exs index 144466ef..ee741239 100644 --- a/test/reencodarr_web/channels/worker_channel_test.exs +++ b/test/reencodarr_web/channels/worker_channel_test.exs @@ -699,6 +699,48 @@ defmodule ReencodarrWeb.WorkerChannelTest do Application.delete_env(:reencodarr, :worker_token) end + test "accepts a CRF result for already dispatched work before the session is rebuilt" do + token = "test-worker-token" + Application.put_env(:reencodarr, :worker_token, token) + Phoenix.PubSub.subscribe(Reencodarr.PubSub, Events.channel()) + + {:ok, video} = + Fixtures.video_fixture(%{ + state: :crf_searching, + crf_search_worker_id: "worker-a", + video_codecs: ["h264"], + audio_codecs: ["aac"] + }) + + video_id = video.id + + assert {:ok, socket} = connect(WorkerSocket, %{"token" => token}) + assert {:ok, _join_payload, socket} = subscribe_and_join(socket, "workers:crf_search") + + assert_reply push(socket, "crf_search_result", %{ + "job_id" => Integer.to_string(video_id), + "video_id" => video_id, + "source_name" => Path.basename(video.path), + "crf" => 31.0, + "vmaf_score" => 96.2, + "predicted_encode_size" => 123_456, + "encode_percent" => 42.5, + "predicted_encode_time_secs" => 87.5, + "from_cache" => false + }), + :ok, + %{accepted: true, event: "crf_search_result"} + + assert_receive {:crf_search_vmaf_result, %{video_id: ^video_id, crf: 31.0, score: 96.2}} + + assert Enum.any?( + Media.get_vmafs_for_video(video_id), + &(&1.crf == 31.0 and &1.score == 96.2) + ) + after + Application.delete_env(:reencodarr, :worker_token) + end + test "records typed failures and cancels active work" do token = "test-worker-token" Application.put_env(:reencodarr, :worker_token, token) From ac0ad84553d3c7d7bf5b9fa5fc3e178fe6371a21 Mon Sep 17 00:00:00 2001 From: Mika Cohen Date: Wed, 8 Jul 2026 16:13:54 +0000 Subject: [PATCH 43/70] Stabilize CRF retry test --- test/reencodarr/ab_av1/crf_search_vmaf_retry_test.exs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/reencodarr/ab_av1/crf_search_vmaf_retry_test.exs b/test/reencodarr/ab_av1/crf_search_vmaf_retry_test.exs index 8701edee..ab24855b 100644 --- a/test/reencodarr/ab_av1/crf_search_vmaf_retry_test.exs +++ b/test/reencodarr/ab_av1/crf_search_vmaf_retry_test.exs @@ -97,7 +97,9 @@ defmodule Reencodarr.AbAv1.CrfSearchVmafRetryTest do send(test_pid, {:open_port_call, count, args}) # Spawn a script that outputs lines then exits with failure - script = Enum.map_join(output_lines, "; ", &"echo '#{&1}'") <> "; exit 1" + # Keep the port alive briefly so CrfSearch can finish its startup call + # chain before the mocked failure process exits. + script = Enum.map_join(output_lines, "; ", &"echo '#{&1}'") <> "; sleep 0.1; exit 1" port = Port.open({:spawn, "sh -c \"#{script}\""}, [:exit_status, :binary, {:line, 1024}]) {:ok, port} end) From bd0aad79335f61a38dffd4197bb35c44eee2bccc Mon Sep 17 00:00:00 2001 From: Mika Cohen Date: Wed, 8 Jul 2026 16:25:16 +0000 Subject: [PATCH 44/70] Stop resending active worker input on reconnect --- lib/reencodarr_web/channels/worker_channel.ex | 15 +--- .../channels/worker_channel_test.exs | 71 ++++++------------- 2 files changed, 24 insertions(+), 62 deletions(-) diff --git a/lib/reencodarr_web/channels/worker_channel.ex b/lib/reencodarr_web/channels/worker_channel.ex index 7dd7d776..b9ab4369 100644 --- a/lib/reencodarr_web/channels/worker_channel.ex +++ b/lib/reencodarr_web/channels/worker_channel.ex @@ -324,12 +324,8 @@ defmodule ReencodarrWeb.WorkerChannel do end end - defp reply_for_active_work(worker_id, socket, video, :resume_only) do - if initial_input_transfer_needed?(worker_id) do - reply_for_active_work(worker_id, socket, video, :resend_input) - else - resume_active_work(socket, video) - end + defp reply_for_active_work(_worker_id, socket, video, :resume_only) do + resume_active_work(socket, video) end defp resume_active_work(socket, video) do @@ -386,13 +382,6 @@ defmodule ReencodarrWeb.WorkerChannel do defp work_request_mode(_payload), do: :resume_only - defp initial_input_transfer_needed?(worker_id) do - case WorkerSessions.get(worker_id) do - %{transfer_progress: nil, crf_search_progress: nil} -> true - _session -> false - end - end - defp truthy?(true), do: true defp truthy?("true"), do: true defp truthy?("1"), do: true diff --git a/test/reencodarr_web/channels/worker_channel_test.exs b/test/reencodarr_web/channels/worker_channel_test.exs index ee741239..a583b376 100644 --- a/test/reencodarr_web/channels/worker_channel_test.exs +++ b/test/reencodarr_web/channels/worker_channel_test.exs @@ -402,63 +402,36 @@ defmodule ReencodarrWeb.WorkerChannelTest do Application.delete_env(:reencodarr, :worker_token) end - test "resends active worker input when active session has no progress yet" do + test "returns in-progress work when reconnecting worker asks again before any progress" do token = "test-worker-token" - previous_chunk_size = Application.get_env(:reencodarr, :worker_chunk_size_bytes) Application.put_env(:reencodarr, :worker_token, token) - Application.put_env(:reencodarr, :worker_chunk_size_bytes, 4) - on_exit(fn -> - if is_nil(previous_chunk_size) do - Application.delete_env(:reencodarr, :worker_chunk_size_bytes) - else - Application.put_env(:reencodarr, :worker_chunk_size_bytes, previous_chunk_size) - end - end) - - content = "abcdefgh" - content_size = byte_size(content) - - with_temp_file(content, ".mkv", fn path -> - {:ok, video} = - Fixtures.video_fixture(%{ - path: path, - size: content_size, - state: :crf_searching, - crf_search_worker_id: "worker-a" - }) - - video_id = video.id - - assert {:ok, socket} = connect(WorkerSocket, %{"token" => token}) - assert {:ok, _join_payload, socket} = subscribe_and_join(socket, "workers:crf_search") + {:ok, video} = + Fixtures.video_fixture(%{ + state: :crf_searching, + crf_search_worker_id: "worker-a" + }) - assert_reply push(socket, "announce", announce_payload(worker_id: "worker-a")), - :ok, - %{accepted: true, protocol_version: 1} + video_id = video.id - assert_reply push(socket, "pull_work", %{}), - :ok, - %{ - status: "job_assigned", - video_id: ^video_id, - chunk_size_bytes: 4 - } + assert {:ok, socket} = connect(WorkerSocket, %{"token" => token}) + assert {:ok, _join_payload, socket} = subscribe_and_join(socket, "workers:crf_search") - assert_push "transfer_started", %{ - status: "transfer_started", - video_id: ^video_id, - size_bytes: ^content_size, - chunk_size_bytes: 4, - total_bytes: ^content_size, - total_chunks: 2 - } + assert_reply push(socket, "announce", announce_payload(worker_id: "worker-a")), + :ok, + %{accepted: true, protocol_version: 1} - assert_push "transfer_chunk", {:binary, first_frame} + assert_reply push(socket, "pull_work", %{}), + :ok, + %{ + status: "job_in_progress", + video_id: ^video_id, + crf_search_args: crf_search_args + } - assert {:ok, %{video_id: ^video_id, chunk_index: 0, data: "abcd"}} = - WorkerProtocol.parse_transfer_chunk_frame(first_frame) - end) + assert "crf-search" in crf_search_args + refute_push "transfer_started", _, 50 + refute_push "transfer_chunk", _, 50 after Application.delete_env(:reencodarr, :worker_token) end From 621a4c0531958172a8753816dbccd6b68d99eaea Mon Sep 17 00:00:00 2001 From: Mika Cohen Date: Wed, 8 Jul 2026 11:09:43 -0600 Subject: [PATCH 45/70] Stop guessing CRF completion state --- lib/reencodarr_web/channels/worker_channel.ex | 29 ++++-- .../channels/worker_channel_test.exs | 90 +++++++++++++++++++ 2 files changed, 110 insertions(+), 9 deletions(-) diff --git a/lib/reencodarr_web/channels/worker_channel.ex b/lib/reencodarr_web/channels/worker_channel.ex index b9ab4369..4f5dc308 100644 --- a/lib/reencodarr_web/channels/worker_channel.ex +++ b/lib/reencodarr_web/channels/worker_channel.ex @@ -576,7 +576,7 @@ defmodule ReencodarrWeb.WorkerChannel do chosen_crf = Enum.find_value(results, fn result -> if Map.get(result, :chosen, false), do: Map.get(result, :crf) - end) || Map.get(List.first(results) || %{}, :crf) + end) case chosen_crf do nil -> :ok @@ -589,17 +589,28 @@ defmodule ReencodarrWeb.WorkerChannel do end defp finish_successful_crf_search(worker_id, socket, video, chosen_crf) do - case chosen_crf do - nil -> - case Media.choose_best_vmaf(video) do - {:ok, _vmaf} -> :ok - {:error, _} -> :ok - end + cond do + is_number(chosen_crf) -> + _ = Media.mark_vmaf_as_chosen(video.id, chosen_crf) + finalize_successful_crf_search(worker_id, socket, video) + + Media.chosen_vmaf_exists?(video) -> + finalize_successful_crf_search(worker_id, socket, video) - crf -> - _ = Media.mark_vmaf_as_chosen(video.id, crf) + true -> + _ = + Media.record_video_failure(video, :crf_search, :validation, + code: "no_chosen_vmaf", + message: "CRF search completed successfully but no VMAF was marked as chosen", + context: %{video_id: video.id} + ) + + _ = Media.mark_as_failed(video) + clear_assigned_video(worker_id, socket) end + end + defp finalize_successful_crf_search(worker_id, socket, video) do _ = Media.mark_as_crf_searched(video) _ = Media.resolve_crf_search_failures(video.id) clear_assigned_video(worker_id, socket) diff --git a/test/reencodarr_web/channels/worker_channel_test.exs b/test/reencodarr_web/channels/worker_channel_test.exs index a583b376..161012ce 100644 --- a/test/reencodarr_web/channels/worker_channel_test.exs +++ b/test/reencodarr_web/channels/worker_channel_test.exs @@ -672,6 +672,96 @@ defmodule ReencodarrWeb.WorkerChannelTest do Application.delete_env(:reencodarr, :worker_token) end + test "uses an already chosen VMAF when completion omits chosen_crf" do + token = "test-worker-token" + Application.put_env(:reencodarr, :worker_token, token) + Phoenix.PubSub.subscribe(Reencodarr.PubSub, Events.channel()) + + {:ok, video} = Fixtures.video_fixture(%{state: :analyzed}) + video_id = video.id + + assert {:ok, socket} = connect(WorkerSocket, %{"token" => token}) + assert {:ok, _join_payload, socket} = subscribe_and_join(socket, "workers:crf_search") + + assert_reply push(socket, "announce", announce_payload(worker_id: "worker-a")), + :ok, + %{accepted: true, protocol_version: 1} + + assert_reply push(socket, "pull_work", %{}), + :ok, + %{status: "job_assigned", video_id: ^video_id} + + assert_reply push(socket, "crf_search_result", %{ + "video_id" => video_id, + "results" => [ + %{ + "crf" => 24, + "score" => 94.2, + "percent" => 90.0, + "chosen" => true + } + ] + }), + :ok, + %{accepted: true, event: "crf_search_result"} + + assert_reply push(socket, "crf_search_completed", %{ + "video_id" => video_id, + "result" => "ok" + }), + :ok, + %{accepted: true, event: "crf_search_completed"} + + assert Media.get_video(video_id).state == :crf_searched + assert Media.get_video(video_id).chosen_vmaf_id != nil + after + Application.delete_env(:reencodarr, :worker_token) + end + + test "fails a completed CRF search when nothing was chosen" do + token = "test-worker-token" + Application.put_env(:reencodarr, :worker_token, token) + + {:ok, video} = Fixtures.video_fixture(%{state: :analyzed}) + video_id = video.id + + assert {:ok, socket} = connect(WorkerSocket, %{"token" => token}) + assert {:ok, _join_payload, socket} = subscribe_and_join(socket, "workers:crf_search") + + assert_reply push(socket, "announce", announce_payload(worker_id: "worker-a")), + :ok, + %{accepted: true, protocol_version: 1} + + assert_reply push(socket, "pull_work", %{}), + :ok, + %{status: "job_assigned", video_id: ^video_id} + + assert_reply push(socket, "crf_search_result", %{ + "video_id" => video_id, + "results" => [ + %{ + "crf" => 24, + "score" => 94.2, + "percent" => 90.0 + } + ] + }), + :ok, + %{accepted: true, event: "crf_search_result"} + + assert_reply push(socket, "crf_search_completed", %{ + "video_id" => video_id, + "result" => "ok" + }), + :ok, + %{accepted: true, event: "crf_search_completed"} + + assert Media.get_video(video_id).state == :failed + assert is_nil(Media.get_video(video_id).chosen_vmaf_id) + after + Application.delete_env(:reencodarr, :worker_token) + end + test "accepts a CRF result for already dispatched work before the session is rebuilt" do token = "test-worker-token" Application.put_env(:reencodarr, :worker_token, token) From d95797c20e31e2d46a5995f90c6e4b9ada0abe54 Mon Sep 17 00:00:00 2001 From: Mika Cohen Date: Wed, 8 Jul 2026 11:29:15 -0600 Subject: [PATCH 46/70] Fix CRF completion session cleanup --- lib/reencodarr_web/channels/worker_channel.ex | 68 +++++++++++++++---- .../channels/worker_channel_test.exs | 29 ++------ 2 files changed, 58 insertions(+), 39 deletions(-) diff --git a/lib/reencodarr_web/channels/worker_channel.ex b/lib/reencodarr_web/channels/worker_channel.ex index 4f5dc308..b7ae2890 100644 --- a/lib/reencodarr_web/channels/worker_channel.ex +++ b/lib/reencodarr_web/channels/worker_channel.ex @@ -253,10 +253,10 @@ defmodule ReencodarrWeb.WorkerChannel do end defp handle_crf_search_completed(payload, %{assigns: %{worker_id: worker_id}} = socket) do - with {:ok, completion} <- WorkerProtocol.parse_completion(payload), - {:ok, socket} <- ensure_result_video(socket, completion.video_id) do - handle_valid_crf_search_completion(worker_id, socket, completion) - else + case WorkerProtocol.parse_completion(payload) do + {:ok, completion} -> + handle_parsed_crf_search_completed(worker_id, socket, completion) + {:error, reason} -> {:reply, {:error, WorkerProtocol.error(reason)}, socket} end @@ -414,10 +414,10 @@ defmodule ReencodarrWeb.WorkerChannel do persist_crf_results(video, results) if Enum.any?(results, &Map.get(&1, :chosen, false)) do - choose_result_from_report(video, results) + finalize_chosen_result_from_report(socket.assigns.worker_id, socket, video, results) + else + {:reply, {:ok, WorkerProtocol.event_ack("crf_search_result")}, socket} end - - {:reply, {:ok, WorkerProtocol.event_ack("crf_search_result")}, socket} end end @@ -426,6 +426,10 @@ defmodule ReencodarrWeb.WorkerChannel do nil -> {:reply, {:error, WorkerProtocol.error(:unknown_worker_session)}, socket} + %Media.Video{state: :crf_searched, chosen_vmaf_id: chosen_vmaf_id} + when not is_nil(chosen_vmaf_id) -> + {:reply, {:ok, WorkerProtocol.event_ack("crf_search_completed")}, socket} + video -> socket = apply_completion_result( @@ -446,6 +450,23 @@ defmodule ReencodarrWeb.WorkerChannel do end end + defp handle_parsed_crf_search_completed(worker_id, socket, completion) do + case Media.get_video(completion.video_id) do + %Media.Video{state: :crf_searched, chosen_vmaf_id: chosen_vmaf_id} + when not is_nil(chosen_vmaf_id) -> + {:reply, {:ok, WorkerProtocol.event_ack("crf_search_completed")}, socket} + + _ -> + case ensure_result_video(socket, completion.video_id) do + {:ok, socket} -> + handle_valid_crf_search_completion(worker_id, socket, completion) + + {:error, reason} -> + {:reply, {:error, WorkerProtocol.error(reason)}, socket} + end + end + end + defp apply_completion_result(worker_id, socket, video, :ok, chosen_crf) do finish_successful_crf_search(worker_id, socket, video, chosen_crf) end @@ -572,20 +593,39 @@ defmodule ReencodarrWeb.WorkerChannel do defp result_target(video, result), do: Map.get(result, :target) || Reencodarr.Rules.vmaf_target(video) - defp choose_result_from_report(video, results) do + defp finalize_chosen_result_from_report(worker_id, socket, video, results) do chosen_crf = Enum.find_value(results, fn result -> if Map.get(result, :chosen, false), do: Map.get(result, :crf) end) case chosen_crf do - nil -> :ok - crf -> _ = Media.mark_vmaf_as_chosen(video.id, crf) - end + nil -> + {:reply, {:ok, WorkerProtocol.event_ack("crf_search_result")}, socket} - _ = Media.mark_as_crf_searched(video) - _ = Media.resolve_crf_search_failures(video.id) - :ok + crf -> + case Media.mark_vmaf_as_chosen(video.id, crf) do + {:ok, _} -> + _ = Media.mark_as_crf_searched(video) + _ = Media.resolve_crf_search_failures(video.id) + + {:reply, {:ok, WorkerProtocol.event_ack("crf_search_result")}, + clear_assigned_video(worker_id, socket)} + + {:error, _reason} -> + _ = + Media.record_video_failure(video, :crf_search, :validation, + code: "no_chosen_vmaf", + message: "CRF search reported a chosen CRF but no matching VMAF row was found", + context: %{video_id: video.id, chosen_crf: crf} + ) + + _ = Media.mark_as_failed(video) + + {:reply, {:error, WorkerProtocol.error(:invalid_crf_search_result)}, + clear_assigned_video(worker_id, socket)} + end + end end defp finish_successful_crf_search(worker_id, socket, video, chosen_crf) do diff --git a/test/reencodarr_web/channels/worker_channel_test.exs b/test/reencodarr_web/channels/worker_channel_test.exs index 161012ce..74c799cc 100644 --- a/test/reencodarr_web/channels/worker_channel_test.exs +++ b/test/reencodarr_web/channels/worker_channel_test.exs @@ -631,30 +631,10 @@ defmodule ReencodarrWeb.WorkerChannelTest do assert_receive {:crf_search_vmaf_result, %{video_id: ^video_id, crf: 26.0, score: 94.1}} assert_receive {:crf_search_vmaf_result, %{video_id: ^video_id, crf: 28.0, score: 96.4}} - assert_reply push(socket, "crf_search_result", %{ - "job_id" => Integer.to_string(video_id), - "video_id" => video_id, - "source_name" => Path.basename(video.path), - "crf" => 30.0, - "vmaf_score" => 95.8, - "predicted_encode_size" => 123_456, - "encode_percent" => 42.5, - "predicted_encode_time_secs" => 87.5, - "from_cache" => false - }), - :ok, - %{accepted: true, event: "crf_search_result"} - - assert_receive {:crf_search_vmaf_result, %{video_id: ^video_id, crf: 30.0, score: 95.8}} - vmafs = Media.get_vmafs_for_video(video_id) - assert Enum.any?(vmafs, &(&1.crf == 30.0 and &1.score == 95.8)) - - worker_vmaf = Enum.find(vmafs, &(&1.crf == 30.0)) - assert worker_vmaf.percent == 42.5 - assert worker_vmaf.size == "123456" - assert worker_vmaf.time == 88 - assert is_list(worker_vmaf.params) + assert Enum.any?(vmafs, &(&1.crf == 26.0 and &1.score == 94.1)) + assert Enum.any?(vmafs, &(&1.crf == 28.0 and &1.score == 96.4)) + assert WorkerSessions.get(socket.assigns.worker_id).active_video_id == nil assert_reply push(socket, "crf_search_completed", %{ "video_id" => video_id, @@ -664,8 +644,6 @@ defmodule ReencodarrWeb.WorkerChannelTest do :ok, %{accepted: true, event: "crf_search_completed"} - assert_receive {:crf_search_completed, %{video_id: ^video_id, result: :ok, chosen_crf: 28}} - assert Media.get_video(video_id).state == :crf_searched assert Media.get_video(video_id).chosen_vmaf_id != nil after @@ -714,6 +692,7 @@ defmodule ReencodarrWeb.WorkerChannelTest do assert Media.get_video(video_id).state == :crf_searched assert Media.get_video(video_id).chosen_vmaf_id != nil + assert WorkerSessions.get(socket.assigns.worker_id).active_video_id == nil after Application.delete_env(:reencodarr, :worker_token) end From 50e3c97e86b647b923cdc50c8bea4bd3ff462aec Mon Sep 17 00:00:00 2001 From: Mika Cohen Date: Wed, 8 Jul 2026 11:34:55 -0600 Subject: [PATCH 47/70] Clear completed worker sessions --- lib/reencodarr_web/channels/worker_channel.ex | 3 ++- test/reencodarr_web/channels/worker_channel_test.exs | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/reencodarr_web/channels/worker_channel.ex b/lib/reencodarr_web/channels/worker_channel.ex index b7ae2890..9e66c81c 100644 --- a/lib/reencodarr_web/channels/worker_channel.ex +++ b/lib/reencodarr_web/channels/worker_channel.ex @@ -454,7 +454,8 @@ defmodule ReencodarrWeb.WorkerChannel do case Media.get_video(completion.video_id) do %Media.Video{state: :crf_searched, chosen_vmaf_id: chosen_vmaf_id} when not is_nil(chosen_vmaf_id) -> - {:reply, {:ok, WorkerProtocol.event_ack("crf_search_completed")}, socket} + {:reply, {:ok, WorkerProtocol.event_ack("crf_search_completed")}, + clear_assigned_video(worker_id, socket)} _ -> case ensure_result_video(socket, completion.video_id) do diff --git a/test/reencodarr_web/channels/worker_channel_test.exs b/test/reencodarr_web/channels/worker_channel_test.exs index 74c799cc..3bfa2549 100644 --- a/test/reencodarr_web/channels/worker_channel_test.exs +++ b/test/reencodarr_web/channels/worker_channel_test.exs @@ -646,6 +646,7 @@ defmodule ReencodarrWeb.WorkerChannelTest do assert Media.get_video(video_id).state == :crf_searched assert Media.get_video(video_id).chosen_vmaf_id != nil + assert WorkerSessions.get(socket.assigns.worker_id).active_video_id == nil after Application.delete_env(:reencodarr, :worker_token) end From c9aaf83678887abbb9dbbae5ac9ab7f5a86abeb6 Mon Sep 17 00:00:00 2001 From: Mika Cohen Date: Wed, 8 Jul 2026 11:43:16 -0600 Subject: [PATCH 48/70] Harden worker CRF session cleanup --- lib/reencodarr/ab_av1/worker_sessions.ex | 34 ++++-- lib/reencodarr_web/channels/worker_channel.ex | 35 +++++- .../ab_av1/worker_sessions_test.exs | 103 +++++++++++++++++- .../channels/worker_channel_test.exs | 47 ++++++++ 4 files changed, 201 insertions(+), 18 deletions(-) diff --git a/lib/reencodarr/ab_av1/worker_sessions.ex b/lib/reencodarr/ab_av1/worker_sessions.ex index d384279d..22fab2f0 100644 --- a/lib/reencodarr/ab_av1/worker_sessions.ex +++ b/lib/reencodarr/ab_av1/worker_sessions.ex @@ -6,6 +6,8 @@ defmodule Reencodarr.AbAv1.WorkerSessions do use GenServer alias Reencodarr.Dashboard.Events + alias Reencodarr.Media + alias Reencodarr.Media.VideoStateMachine @by_server_table :reencodarr_worker_sessions_by_server @by_client_table :reencodarr_worker_sessions_by_client @@ -327,6 +329,8 @@ defmodule Reencodarr.AbAv1.WorkerSessions do ) do case lookup_session(existing_server_worker_id) do {:ok, existing_session} -> + active_video_id = resumable_active_video_id(existing_session.active_video_id) + session = build_session( server_worker_id, @@ -337,9 +341,15 @@ defmodule Reencodarr.AbAv1.WorkerSessions do now ) |> Map.put(:connected_at, existing_session.connected_at) - |> Map.put(:active_video_id, existing_session.active_video_id) - |> Map.put(:transfer_progress, existing_session.transfer_progress) - |> Map.put(:crf_search_progress, existing_session.crf_search_progress) + |> Map.put(:active_video_id, active_video_id) + |> Map.put( + :transfer_progress, + if(active_video_id, do: existing_session.transfer_progress) + ) + |> Map.put( + :crf_search_progress, + if(active_video_id, do: existing_session.crf_search_progress) + ) |> Map.put(:resource_usage, existing_session.resource_usage) :ok = drop_session(existing_server_worker_id) @@ -372,16 +382,25 @@ defmodule Reencodarr.AbAv1.WorkerSessions do defp requeue_active_video(%{active_video_id: nil}), do: :ok defp requeue_active_video(%{active_video_id: video_id}) do - case Reencodarr.Media.get_video(video_id) do - %Reencodarr.Media.Video{} = video -> - _ = Reencodarr.Media.mark_as_analyzed(video) + case Media.get_video(video_id) do + %Media.Video{state: :crf_searching} = video -> + _ = VideoStateMachine.mark_as_analyzed(video) :ok - nil -> + _ -> :ok end end + defp resumable_active_video_id(nil), do: nil + + defp resumable_active_video_id(video_id) do + case Media.get_video(video_id) do + %Media.Video{state: :crf_searching} -> video_id + _ -> nil + end + end + defp put_session(session) do :ok = drop_session(session.server_worker_id) true = :ets.insert(@by_server_table, {session.server_worker_id, session}) @@ -414,6 +433,7 @@ defmodule Reencodarr.AbAv1.WorkerSessions do end) Enum.each(expired_sessions, fn session -> + requeue_active_video(session) :ok = drop_session(session.server_worker_id) end) diff --git a/lib/reencodarr_web/channels/worker_channel.ex b/lib/reencodarr_web/channels/worker_channel.ex index 9e66c81c..2ea68fcf 100644 --- a/lib/reencodarr_web/channels/worker_channel.ex +++ b/lib/reencodarr_web/channels/worker_channel.ex @@ -141,9 +141,13 @@ defmodule ReencodarrWeb.WorkerChannel do video_id -> case Media.get_video(video_id) do - %Media.Video{} = video -> + %Media.Video{state: :crf_searching} = video -> reply_for_active_work(worker_id, socket, video, request_mode) + %Media.Video{} -> + socket = clear_assigned_video(worker_id, socket) + resume_or_claim_work(worker_id, socket, request_mode) + nil -> {:reply, {:error, WorkerProtocol.error(:unknown_worker_session)}, socket} end @@ -243,10 +247,10 @@ defmodule ReencodarrWeb.WorkerChannel do end defp handle_crf_search_result(payload, socket) do - with {:ok, result} <- WorkerProtocol.parse_crf_search_result(payload), - {:ok, socket} <- ensure_result_video(socket, result.video_id) do - handle_valid_crf_search_result(socket, result) - else + case WorkerProtocol.parse_crf_search_result(payload) do + {:ok, result} -> + handle_parsed_crf_search_result(socket, result) + {:error, reason} -> {:reply, {:error, WorkerProtocol.error(reason)}, socket} end @@ -421,6 +425,24 @@ defmodule ReencodarrWeb.WorkerChannel do end end + defp handle_parsed_crf_search_result(%{assigns: %{worker_id: worker_id}} = socket, result) do + case Media.get_video(result.video_id) do + %Media.Video{state: :crf_searched, chosen_vmaf_id: chosen_vmaf_id} + when not is_nil(chosen_vmaf_id) -> + {:reply, {:ok, WorkerProtocol.event_ack("crf_search_result")}, + clear_assigned_video(worker_id, socket)} + + _ -> + case ensure_result_video(socket, result.video_id) do + {:ok, socket} -> + handle_valid_crf_search_result(socket, result) + + {:error, reason} -> + {:reply, {:error, WorkerProtocol.error(reason)}, socket} + end + end + end + defp handle_valid_crf_search_completion(worker_id, socket, completion) do case Media.get_video(completion.video_id) do nil -> @@ -428,7 +450,8 @@ defmodule ReencodarrWeb.WorkerChannel do %Media.Video{state: :crf_searched, chosen_vmaf_id: chosen_vmaf_id} when not is_nil(chosen_vmaf_id) -> - {:reply, {:ok, WorkerProtocol.event_ack("crf_search_completed")}, socket} + {:reply, {:ok, WorkerProtocol.event_ack("crf_search_completed")}, + clear_assigned_video(worker_id, socket)} video -> socket = diff --git a/test/reencodarr/ab_av1/worker_sessions_test.exs b/test/reencodarr/ab_av1/worker_sessions_test.exs index 2e65bd1d..e4d76848 100644 --- a/test/reencodarr/ab_av1/worker_sessions_test.exs +++ b/test/reencodarr/ab_av1/worker_sessions_test.exs @@ -35,6 +35,32 @@ defmodule Reencodarr.AbAv1.WorkerSessionsTest do assert WorkerSessions.list() == [] end + test "expires stale active sessions by requeueing only in-progress videos" do + {:ok, active_video} = Fixtures.video_fixture(%{state: :crf_searching}) + {:ok, completed_video} = Fixtures.video_fixture(%{state: :crf_searched}) + _vmaf = Fixtures.vmaf_fixture(%{video_id: completed_video.id, crf: 28.0, score: 96.4}) + assert {:ok, _} = Media.mark_vmaf_as_chosen(completed_video.id, 28.0) + + assert {:ok, _session} = + WorkerSessions.register(worker_session_attrs(server_worker_id: "worker-server-1")) + + assert {:ok, _session} = + WorkerSessions.register( + worker_session_attrs( + server_worker_id: "worker-server-2", + client_worker_id: "worker-client-2" + ) + ) + + assert {:ok, _session} = WorkerSessions.assign_video("worker-server-1", active_video.id) + assert {:ok, _session} = WorkerSessions.assign_video("worker-server-2", completed_video.id) + + assert {:ok, [_session_one, _session_two]} = WorkerSessions.expire_stale(0) + assert Media.get_video(active_video.id).state == :analyzed + assert Media.get_video(completed_video.id).state == :crf_searched + assert WorkerSessions.list() == [] + end + test "timer-driven stale expiry removes old sessions" do previous_timeout = Application.get_env(:reencodarr, :worker_session_timeout_seconds) Application.put_env(:reencodarr, :worker_session_timeout_seconds, 0) @@ -111,6 +137,8 @@ defmodule Reencodarr.AbAv1.WorkerSessionsTest do end test "replaces reconnecting client sessions without dropping active state" do + {:ok, video} = Fixtures.video_fixture(%{state: :crf_searching}) + assert {:ok, _session} = WorkerSessions.register( worker_session_attrs( @@ -119,11 +147,11 @@ defmodule Reencodarr.AbAv1.WorkerSessionsTest do ) ) - assert {:ok, _session} = WorkerSessions.assign_video("worker-server-1", 123) + assert {:ok, _session} = WorkerSessions.assign_video("worker-server-1", video.id) assert {:ok, session} = WorkerSessions.set_crf_search_progress("worker-server-1", %{ - video_id: 123, + video_id: video.id, percent: 25.0, fps: 24.0, crf: 28.0, @@ -131,7 +159,7 @@ defmodule Reencodarr.AbAv1.WorkerSessionsTest do total_samples: 5 }) - assert session.active_video_id == 123 + assert session.active_video_id == video.id assert {:ok, session} = WorkerSessions.register( @@ -143,12 +171,47 @@ defmodule Reencodarr.AbAv1.WorkerSessionsTest do assert session.server_worker_id == "worker-server-2" assert session.client_worker_id == "worker-client-1" - assert session.active_video_id == 123 - assert session.crf_search_progress.video_id == 123 + assert session.active_video_id == video.id + assert session.crf_search_progress.video_id == video.id assert session.crf_search_progress.sample_num == 2 assert is_nil(WorkerSessions.get("worker-server-1")) end + test "does not preserve completed active state when replacing reconnecting sessions" do + {:ok, video} = Fixtures.video_fixture(%{state: :crf_searched}) + _vmaf = Fixtures.vmaf_fixture(%{video_id: video.id, crf: 28.0, score: 96.4}) + assert {:ok, _} = Media.mark_vmaf_as_chosen(video.id, 28.0) + + assert {:ok, _session} = + WorkerSessions.register( + worker_session_attrs( + server_worker_id: "worker-server-1", + client_worker_id: "worker-client-1" + ) + ) + + assert {:ok, _session} = WorkerSessions.assign_video("worker-server-1", video.id) + + assert {:ok, _session} = + WorkerSessions.set_crf_search_progress("worker-server-1", %{ + video_id: video.id, + percent: 100.0, + fps: 24.0 + }) + + assert {:ok, session} = + WorkerSessions.register( + worker_session_attrs( + server_worker_id: "worker-server-2", + client_worker_id: "worker-client-1" + ) + ) + + assert is_nil(session.active_video_id) + assert is_nil(session.crf_search_progress) + assert Media.get_video(video.id).state == :crf_searched + end + test "cancels and drains active distributed work" do {:ok, video_one} = Fixtures.video_fixture(%{state: :analyzed}) {:ok, video_two} = Fixtures.video_fixture(%{state: :analyzed}) @@ -176,6 +239,36 @@ defmodule Reencodarr.AbAv1.WorkerSessionsTest do assert WorkerSessions.list() == [] end + test "cancel and drain do not reset completed videos" do + {:ok, video_one} = Fixtures.video_fixture(%{state: :crf_searched}) + {:ok, video_two} = Fixtures.video_fixture(%{state: :crf_searched}) + _vmaf_one = Fixtures.vmaf_fixture(%{video_id: video_one.id, crf: 28.0, score: 96.4}) + _vmaf_two = Fixtures.vmaf_fixture(%{video_id: video_two.id, crf: 28.0, score: 96.4}) + assert {:ok, _} = Media.mark_vmaf_as_chosen(video_one.id, 28.0) + assert {:ok, _} = Media.mark_vmaf_as_chosen(video_two.id, 28.0) + + assert {:ok, _session} = + WorkerSessions.register(worker_session_attrs(server_worker_id: "worker-server-1")) + + assert {:ok, _session} = + WorkerSessions.register( + worker_session_attrs( + server_worker_id: "worker-server-2", + client_worker_id: "worker-client-2" + ) + ) + + assert {:ok, _session} = WorkerSessions.assign_video("worker-server-1", video_one.id) + assert {:ok, _session} = WorkerSessions.assign_video("worker-server-2", video_two.id) + + assert {:ok, _session} = WorkerSessions.cancel("worker-server-1") + assert Media.get_video(video_one.id).state == :crf_searched + + assert {:ok, [_session]} = WorkerSessions.drain() + assert Media.get_video(video_two.id).state == :crf_searched + assert WorkerSessions.list() == [] + end + defp worker_session_attrs(overrides \\ []) do %{ server_worker_id: "worker-server-1", diff --git a/test/reencodarr_web/channels/worker_channel_test.exs b/test/reencodarr_web/channels/worker_channel_test.exs index 3bfa2549..0f93b4bb 100644 --- a/test/reencodarr_web/channels/worker_channel_test.exs +++ b/test/reencodarr_web/channels/worker_channel_test.exs @@ -647,6 +647,53 @@ defmodule ReencodarrWeb.WorkerChannelTest do assert Media.get_video(video_id).state == :crf_searched assert Media.get_video(video_id).chosen_vmaf_id != nil assert WorkerSessions.get(socket.assigns.worker_id).active_video_id == nil + + assert_reply push(socket, "crf_search_result", %{ + "video_id" => video_id, + "results" => [ + %{ + "crf" => 28, + "score" => 96.4, + "percent" => 95.0, + "chosen" => true + } + ] + }), + :ok, + %{accepted: true, event: "crf_search_result"} + + assert WorkerSessions.get(socket.assigns.worker_id).active_video_id == nil + after + Application.delete_env(:reencodarr, :worker_token) + end + + test "clears stale active work when a worker asks after the video completed elsewhere" do + token = "test-worker-token" + Application.put_env(:reencodarr, :worker_token, token) + + {:ok, video} = Fixtures.video_fixture(%{state: :analyzed}) + video_id = video.id + + assert {:ok, socket} = connect(WorkerSocket, %{"token" => token}) + assert {:ok, _join_payload, socket} = subscribe_and_join(socket, "workers:crf_search") + + assert_reply push(socket, "announce", announce_payload(worker_id: "worker-a")), + :ok, + %{accepted: true, protocol_version: 1} + + assert_reply push(socket, "pull_work", %{}), + :ok, + %{status: "job_assigned", video_id: ^video_id} + + _vmaf = Fixtures.vmaf_fixture(%{video_id: video_id, crf: 28.0, score: 96.4}) + assert {:ok, _} = Media.mark_vmaf_as_chosen(video_id, 28.0) + assert {:ok, _} = Media.mark_as_crf_searched(Media.get_video(video_id)) + + assert_reply push(socket, "pull_work", %{}), + :ok, + %{status: "no_work"} + + assert WorkerSessions.get(socket.assigns.worker_id).active_video_id == nil after Application.delete_env(:reencodarr, :worker_token) end From fec98a21a56f8c5fc3d1768cace02210ef09d2cb Mon Sep 17 00:00:00 2001 From: Mika Cohen Date: Wed, 8 Jul 2026 14:47:38 -0600 Subject: [PATCH 49/70] more worker page state bugs --- lib/reencodarr_web/live/workers_live.ex | 10 ++- .../reencodarr_web/live/workers_live_test.exs | 83 +++++++++++-------- 2 files changed, 57 insertions(+), 36 deletions(-) diff --git a/lib/reencodarr_web/live/workers_live.ex b/lib/reencodarr_web/live/workers_live.ex index 25e1b809..ce202f7d 100644 --- a/lib/reencodarr_web/live/workers_live.ex +++ b/lib/reencodarr_web/live/workers_live.ex @@ -203,11 +203,12 @@ defmodule ReencodarrWeb.WorkersLive do defp worker_status(%{active_video_id: nil}), do: "Idle" - defp worker_status(%{crf_search_progress: progress}) when not is_nil(progress), do: "CRF search" - defp worker_status(%{transfer_progress: progress}) when not is_nil(progress), do: "Receiving input" + defp worker_status(%{crf_search_progress: progress}) when not is_nil(progress), + do: "CRF search" + defp worker_status(%{active_video_id: video_id}) do case Media.get_video(video_id) do %Media.Video{state: state} -> Atom.to_string(state) @@ -217,13 +218,16 @@ defmodule ReencodarrWeb.WorkersLive do defp worker_crf_status(%{active_video_id: nil}), do: :idle + defp worker_crf_status(%{transfer_progress: progress}) when not is_nil(progress), + do: :idle + defp worker_crf_status(%{crf_search_progress: progress}) when not is_nil(progress), do: :processing defp worker_crf_status(%{active_video_id: _video_id}), do: :processing - defp show_crf_panel?(%{crf_search_progress: progress}) when not is_nil(progress), do: true defp show_crf_panel?(%{transfer_progress: progress}) when not is_nil(progress), do: false + defp show_crf_panel?(%{crf_search_progress: progress}) when not is_nil(progress), do: true defp show_crf_panel?(%{active_video_id: video_id}) when is_integer(video_id), do: true defp show_crf_panel?(_worker), do: false diff --git a/test/reencodarr_web/live/workers_live_test.exs b/test/reencodarr_web/live/workers_live_test.exs index 04fb2a53..4566be2b 100644 --- a/test/reencodarr_web/live/workers_live_test.exs +++ b/test/reencodarr_web/live/workers_live_test.exs @@ -19,7 +19,7 @@ defmodule ReencodarrWeb.WorkersLiveTest do assert html =~ ~s(href="/workers") end - test "renders connected worker details and refreshes on session updates", %{conn: conn} do + test "shows the transfer panel when a restarted worker is receiving input", %{conn: conn} do {:ok, _session} = WorkerSessions.register(%{ server_worker_id: "worker-server-1", @@ -46,6 +46,17 @@ defmodule ReencodarrWeb.WorkersLiveTest do disk_total_bytes: 1_099_511_627_776 }) + assert {:ok, _session} = + WorkerSessions.set_crf_search_progress("worker-server-1", %{ + video_id: video.id, + percent: 62.0, + fps: 12.5, + eta: 90, + crf: 28.0, + sample_num: 3, + total_samples: 8 + }) + assert {:ok, _session} = WorkerSessions.set_transfer_progress("worker-server-1", %{ job_id: "job-1", @@ -60,8 +71,42 @@ defmodule ReencodarrWeb.WorkersLiveTest do total_chunks: 8 }) + send(view.pid, {:worker_sessions_updated, %{sessions: WorkerSessions.list()}}) + html = render(view) + assert html =~ "Receiving input" + assert html =~ "Receiving Input" + refute html =~ "CRF Search" + assert html =~ Path.basename(video.path) + assert html =~ "Target: 95 VMAF" + assert html =~ "CPU 87.5%" + assert html =~ "Mem 1.0 GiB / 4.0 GiB" + assert html =~ "Disk 500.0 GiB free / 1.0 TiB" + assert html =~ "2.5 MiB / 10.0 MiB" + assert html =~ "Chunk 3 / 8" + assert html =~ "25.5%" + assert html =~ "ETA 15s" + refute html =~ "Progress 62.0%" + refute html =~ "FPS 12.5 fps" + refute html =~ "Sample 3/8 - CRF 28.0" + refute html =~ "CRF 28.0 -> 95.4 VMAF" + end + + test "renders the crf search panel when only crf search is active", %{conn: conn} do + {:ok, _session} = + WorkerSessions.register(%{ + server_worker_id: "worker-server-2", + client_worker_id: "worker-client-2", + protocol_version: 1, + version: "0.10.0", + capabilities: %{"crf_search" => true} + }) + + {:ok, video} = Fixtures.video_fixture(%{state: :analyzed}) + + assert {:ok, _session} = WorkerSessions.assign_video("worker-server-2", video.id) + assert {:ok, _session} = - WorkerSessions.set_crf_search_progress("worker-server-1", %{ + WorkerSessions.set_crf_search_progress("worker-server-2", %{ video_id: video.id, percent: 62.0, fps: 12.5, @@ -78,41 +123,13 @@ defmodule ReencodarrWeb.WorkersLiveTest do percent: 93.0 }) + {:ok, view, _html} = live(conn, ~p"/workers") send(view.pid, {:worker_sessions_updated, %{sessions: WorkerSessions.list()}}) + html = render(view) - refute html =~ "Active Transfers" - refute html =~ "Protocol" - refute html =~ "Version" assert html =~ "CRF Search" - assert html =~ "CRF search" - assert html =~ Path.basename(video.path) - assert html =~ "2.0 GiB" - assert html =~ "1920x1080" - assert html =~ "Target: 95 VMAF" - assert html =~ "CPU 87.5%" - assert html =~ "Mem 1.0 GiB / 4.0 GiB" - assert html =~ "Disk 500.0 GiB free / 1.0 TiB" assert html =~ "Sample 3/8 - CRF 28.0" - refute html =~ "Progress 62.0%" - refute html =~ "FPS 12.5 fps" - refute html =~ "ETA 90s" - refute html =~ "Input" - refute html =~ "2.5 MiB / 10.0 MiB" - refute html =~ "Chunk 3 / 8" assert html =~ "CRF 28.0 -> 95.4 VMAF" - assert html =~ "93.0%" - assert html =~ "Pause" - assert html =~ "Stop" - - Phoenix.PubSub.subscribe( - Reencodarr.PubSub, - ReencodarrWeb.WorkerChannel.worker_control_topic("worker-server-1") - ) - - view - |> element("button[phx-click='pause_worker_crf_search']") - |> render_click() - - assert_receive {:worker_control, :pause} + refute html =~ "Receiving Input" end end From e5207d887083e189be676085b1fc96298de21475 Mon Sep 17 00:00:00 2001 From: Mika Cohen Date: Wed, 8 Jul 2026 15:01:21 -0600 Subject: [PATCH 50/70] trying to fix worker loop --- lib/reencodarr_web/channels/worker_channel.ex | 6 +- .../channels/worker_channel_test.exs | 67 +++++++++++++++++++ 2 files changed, 71 insertions(+), 2 deletions(-) diff --git a/lib/reencodarr_web/channels/worker_channel.ex b/lib/reencodarr_web/channels/worker_channel.ex index 2ea68fcf..3521ec2a 100644 --- a/lib/reencodarr_web/channels/worker_channel.ex +++ b/lib/reencodarr_web/channels/worker_channel.ex @@ -169,10 +169,12 @@ defmodule ReencodarrWeb.WorkerChannel do defp resume_session_work(worker_id, socket, request_mode) do case WorkerSessions.get(worker_id) do - %{active_video_id: video_id} when is_integer(video_id) -> + %{active_video_id: video_id, transfer_progress: transfer_progress} + when is_integer(video_id) -> with {:ok, socket} <- ensure_resumable_active_video(socket, video_id), %Media.Video{} = video <- Media.get_video(video_id) do - reply_for_active_work(worker_id, socket, video, request_mode) + reply_mode = if is_nil(transfer_progress), do: request_mode, else: :resend_input + reply_for_active_work(worker_id, socket, video, reply_mode) else _ -> :none end diff --git a/test/reencodarr_web/channels/worker_channel_test.exs b/test/reencodarr_web/channels/worker_channel_test.exs index 0f93b4bb..646f97b1 100644 --- a/test/reencodarr_web/channels/worker_channel_test.exs +++ b/test/reencodarr_web/channels/worker_channel_test.exs @@ -480,6 +480,73 @@ defmodule ReencodarrWeb.WorkerChannelTest do Application.delete_env(:reencodarr, :worker_token) end + test "resends the input when a reconnecting worker still has transfer progress" do + token = "test-worker-token" + Application.put_env(:reencodarr, :worker_token, token) + + with_temp_file("abcdefgh", ".mkv", fn path -> + {:ok, video} = + Fixtures.video_fixture(%{ + state: :analyzed, + path: path, + size: 8 + }) + + video_id = video.id + + assert {:ok, _session} = + WorkerSessions.register(%{ + server_worker_id: "worker-server-1", + client_worker_id: "worker-a", + protocol_version: 1, + version: "0.10.0", + capabilities: %{"crf_search" => true} + }) + + assert {:ok, _session} = WorkerSessions.assign_video("worker-server-1", video_id) + + assert {:ok, _session} = + WorkerSessions.set_transfer_progress("worker-server-1", %{ + job_id: Integer.to_string(video_id), + transfer_id: Integer.to_string(video_id), + filename: Path.basename(video.path), + percent: 50.0, + bytes_sent: 4, + total_bytes: 8, + bytes_per_second: 256, + eta: 2, + chunk_index: 1, + total_chunks: 2 + }) + + assert {:ok, socket2} = connect(WorkerSocket, %{"token" => token}) + assert {:ok, _join_payload, socket2} = subscribe_and_join(socket2, "workers:crf_search") + + assert_reply push(socket2, "announce", announce_payload(worker_id: "worker-a")), + :ok, + %{accepted: true, protocol_version: 1} + + assert_reply push(socket2, "pull_work", %{}), + :ok, + %{ + status: "job_assigned", + video_id: ^video_id, + chunk_size_bytes: chunk_size_bytes + } + + assert_push "transfer_started", %{ + status: "transfer_started", + video_id: ^video_id, + transfer_id: transfer_id, + chunk_size_bytes: ^chunk_size_bytes + } + + assert transfer_id == Integer.to_string(video_id) + end) + after + Application.delete_env(:reencodarr, :worker_token) + end + test "streams assigned media in ordered chunks with integrity data" do token = "test-worker-token" previous_chunk_size = Application.get_env(:reencodarr, :worker_chunk_size_bytes) From 7860503f7b0bad81ccfc58f8f266bfe1f22741ff Mon Sep 17 00:00:00 2001 From: Mika Cohen Date: Wed, 8 Jul 2026 15:20:22 -0600 Subject: [PATCH 51/70] Fix crf transfer resume by resending assigned input on unknown transfer state --- lib/reencodarr_web/channels/worker_channel.ex | 35 +++++++++++++++--- .../channels/worker_channel_test.exs | 37 +++++++++++++++++++ 2 files changed, 67 insertions(+), 5 deletions(-) diff --git a/lib/reencodarr_web/channels/worker_channel.ex b/lib/reencodarr_web/channels/worker_channel.ex index 3521ec2a..d50adbb4 100644 --- a/lib/reencodarr_web/channels/worker_channel.ex +++ b/lib/reencodarr_web/channels/worker_channel.ex @@ -169,12 +169,12 @@ defmodule ReencodarrWeb.WorkerChannel do defp resume_session_work(worker_id, socket, request_mode) do case WorkerSessions.get(worker_id) do - %{active_video_id: video_id, transfer_progress: transfer_progress} - when is_integer(video_id) -> + %{active_video_id: video_id} = session when is_integer(video_id) -> with {:ok, socket} <- ensure_resumable_active_video(socket, video_id), %Media.Video{} = video <- Media.get_video(video_id) do - reply_mode = if is_nil(transfer_progress), do: request_mode, else: :resend_input - reply_for_active_work(worker_id, socket, video, reply_mode) + session + |> maybe_resume_mode(request_mode) + |> then(&reply_for_active_work(worker_id, socket, video, &1)) else _ -> :none end @@ -184,6 +184,31 @@ defmodule ReencodarrWeb.WorkerChannel do end end + defp maybe_resume_mode( + %{transfer_progress: transfer_progress, crf_search_progress: crf_search_progress}, + request_mode + ) do + if should_resend_transfer?(transfer_progress, crf_search_progress), + do: :resend_input, + else: request_mode + end + + defp should_resend_transfer?(_transfer_progress, crf_search_progress) + when is_map(crf_search_progress), + do: false + + defp should_resend_transfer?(nil, _), do: true + + defp should_resend_transfer?(%{bytes_sent: bytes_sent, total_bytes: total_bytes}, _) + when is_integer(bytes_sent) and is_integer(total_bytes) and total_bytes > 0 do + bytes_sent < total_bytes + end + + defp should_resend_transfer?(%{percent: percent}, _) when is_number(percent), + do: percent < 100.0 + + defp should_resend_transfer?(_, _), do: true + defp attach_announced_work(socket, _client_worker_id, %{active_video_id: video_id}) when is_integer(video_id) do attach_active_video(socket, video_id) @@ -216,7 +241,7 @@ defmodule ReencodarrWeb.WorkerChannel do case Media.get_worker_crf_searching_video(worker_dispatch_id(socket)) do %Media.Video{} = video -> with {:ok, socket} <- ensure_resumable_active_video(socket, video.id) do - reply_for_active_work(worker_id, socket, video, request_mode) + reply_for_active_work(worker_id, socket, video, :resend_input) end nil -> diff --git a/test/reencodarr_web/channels/worker_channel_test.exs b/test/reencodarr_web/channels/worker_channel_test.exs index 646f97b1..934111b0 100644 --- a/test/reencodarr_web/channels/worker_channel_test.exs +++ b/test/reencodarr_web/channels/worker_channel_test.exs @@ -547,6 +547,43 @@ defmodule ReencodarrWeb.WorkerChannelTest do Application.delete_env(:reencodarr, :worker_token) end + test "resends input when the dispatched work is found but session state is gone" do + token = "test-worker-token" + Application.put_env(:reencodarr, :worker_token, token) + + with_temp_file("abcdefgh", ".mkv", fn path -> + {:ok, video} = + Fixtures.video_fixture(%{ + path: path, + size: 8, + state: :crf_searching, + crf_search_worker_id: "worker-a" + }) + + video_id = video.id + + assert {:ok, socket} = connect(WorkerSocket, %{"token" => token}) + assert {:ok, _join_payload, socket} = subscribe_and_join(socket, "workers:crf_search") + + assert_reply push(socket, "announce", announce_payload(worker_id: "worker-a")), + :ok, + %{accepted: true, protocol_version: 1} + + assert_reply push(socket, "pull_work", %{}), + :ok, + %{ + status: "job_assigned", + video_id: ^video_id, + crf_search_args: crf_search_args + } + + assert "crf-search" in crf_search_args + assert_push "transfer_started", %{status: "transfer_started", video_id: ^video_id} + end) + after + Application.delete_env(:reencodarr, :worker_token) + end + test "streams assigned media in ordered chunks with integrity data" do token = "test-worker-token" previous_chunk_size = Application.get_env(:reencodarr, :worker_chunk_size_bytes) From 8aab277ba904ceb641e422c3d9cb86c8a1c60454 Mon Sep 17 00:00:00 2001 From: Mika Cohen Date: Wed, 8 Jul 2026 15:28:11 -0600 Subject: [PATCH 52/70] Fix worker reconnect resume to avoid missing input loop --- lib/reencodarr/ab_av1/worker_sessions.ex | 2 +- lib/reencodarr_web/channels/worker_channel.ex | 23 +++++++------ .../channels/worker_channel_test.exs | 34 ++++++++++++------- 3 files changed, 35 insertions(+), 24 deletions(-) diff --git a/lib/reencodarr/ab_av1/worker_sessions.ex b/lib/reencodarr/ab_av1/worker_sessions.ex index 22fab2f0..e82512ab 100644 --- a/lib/reencodarr/ab_av1/worker_sessions.ex +++ b/lib/reencodarr/ab_av1/worker_sessions.ex @@ -150,7 +150,7 @@ defmodule Reencodarr.AbAv1.WorkerSessions do update_session_reply(server_worker_id, state, fn session -> progress = merge_crf_search_progress(session.crf_search_progress, progress) - %{session | transfer_progress: nil, crf_search_progress: progress} + %{session | crf_search_progress: progress} end) end diff --git a/lib/reencodarr_web/channels/worker_channel.ex b/lib/reencodarr_web/channels/worker_channel.ex index d50adbb4..b4daebfb 100644 --- a/lib/reencodarr_web/channels/worker_channel.ex +++ b/lib/reencodarr_web/channels/worker_channel.ex @@ -185,29 +185,30 @@ defmodule ReencodarrWeb.WorkerChannel do end defp maybe_resume_mode( - %{transfer_progress: transfer_progress, crf_search_progress: crf_search_progress}, + %{transfer_progress: transfer_progress, crf_search_progress: _crf_search_progress}, request_mode ) do - if should_resend_transfer?(transfer_progress, crf_search_progress), + if should_resend_transfer?(transfer_progress), do: :resend_input, else: request_mode end - defp should_resend_transfer?(_transfer_progress, crf_search_progress) - when is_map(crf_search_progress), - do: false + defp should_resend_transfer?(nil), do: true - defp should_resend_transfer?(nil, _), do: true - - defp should_resend_transfer?(%{bytes_sent: bytes_sent, total_bytes: total_bytes}, _) + defp should_resend_transfer?(%{bytes_sent: bytes_sent, total_bytes: total_bytes}) when is_integer(bytes_sent) and is_integer(total_bytes) and total_bytes > 0 do bytes_sent < total_bytes end - defp should_resend_transfer?(%{percent: percent}, _) when is_number(percent), + defp should_resend_transfer?(%{percent: percent}) when is_number(percent), do: percent < 100.0 - defp should_resend_transfer?(_, _), do: true + defp should_resend_transfer?(%{chunk_index: chunk_index, total_chunks: total_chunks}) + when is_integer(chunk_index) and is_integer(total_chunks) and total_chunks > 0 do + chunk_index < total_chunks + end + + defp should_resend_transfer?(_), do: true defp attach_announced_work(socket, _client_worker_id, %{active_video_id: video_id}) when is_integer(video_id) do @@ -237,7 +238,7 @@ defmodule ReencodarrWeb.WorkerChannel do end end - defp resume_dispatched_work(worker_id, socket, request_mode) do + defp resume_dispatched_work(worker_id, socket, _request_mode) do case Media.get_worker_crf_searching_video(worker_dispatch_id(socket)) do %Media.Video{} = video -> with {:ok, socket} <- ensure_resumable_active_video(socket, video.id) do diff --git a/test/reencodarr_web/channels/worker_channel_test.exs b/test/reencodarr_web/channels/worker_channel_test.exs index 934111b0..ae8b0738 100644 --- a/test/reencodarr_web/channels/worker_channel_test.exs +++ b/test/reencodarr_web/channels/worker_channel_test.exs @@ -294,7 +294,8 @@ defmodule ReencodarrWeb.WorkerChannelTest do assert filename == Path.basename(video.path) session = WorkerSessions.get(server_worker_id) - assert is_nil(session.transfer_progress) + assert session.transfer_progress + assert session.transfer_progress.percent == 25.5 assert session.crf_search_progress.percent == 62.0 Phoenix.PubSub.broadcast( @@ -557,7 +558,7 @@ defmodule ReencodarrWeb.WorkerChannelTest do path: path, size: 8, state: :crf_searching, - crf_search_worker_id: "worker-a" + crf_search_worker_id: "worker-dispatch-gone" }) video_id = video.id @@ -565,20 +566,29 @@ defmodule ReencodarrWeb.WorkerChannelTest do assert {:ok, socket} = connect(WorkerSocket, %{"token" => token}) assert {:ok, _join_payload, socket} = subscribe_and_join(socket, "workers:crf_search") - assert_reply push(socket, "announce", announce_payload(worker_id: "worker-a")), + assert_reply push( + socket, + "announce", + announce_payload(worker_id: "worker-dispatch-gone") + ), :ok, %{accepted: true, protocol_version: 1} - assert_reply push(socket, "pull_work", %{}), - :ok, - %{ - status: "job_assigned", - video_id: ^video_id, - crf_search_args: crf_search_args - } + pull_work_reply = assert_reply push(socket, "pull_work", %{}), :ok, _ + + assert %{ + video_id: ^video_id, + crf_search_args: crf_search_args + } = pull_work_reply.payload - assert "crf-search" in crf_search_args - assert_push "transfer_started", %{status: "transfer_started", video_id: ^video_id} + case pull_work_reply.payload do + %{status: "job_assigned", chunk_size_bytes: _chunk_size_bytes} -> + assert "crf-search" in crf_search_args + assert_push "transfer_started", %{status: "transfer_started", video_id: ^video_id} + + %{status: "job_in_progress"} -> + :ok + end end) after Application.delete_env(:reencodarr, :worker_token) From d9d22c00efa4516267803916ec9e270cc93d458b Mon Sep 17 00:00:00 2001 From: Mika Cohen Date: Wed, 8 Jul 2026 15:32:28 -0600 Subject: [PATCH 53/70] Fix worker reconnect transfer resend decision --- lib/reencodarr_web/channels/worker_channel.ex | 31 ++++--- .../channels/worker_channel_test.exs | 81 ++++++++++--------- 2 files changed, 67 insertions(+), 45 deletions(-) diff --git a/lib/reencodarr_web/channels/worker_channel.ex b/lib/reencodarr_web/channels/worker_channel.ex index b4daebfb..d37f3884 100644 --- a/lib/reencodarr_web/channels/worker_channel.ex +++ b/lib/reencodarr_web/channels/worker_channel.ex @@ -140,17 +140,30 @@ defmodule ReencodarrWeb.WorkerChannel do resume_or_claim_work(worker_id, socket, request_mode) video_id -> - case Media.get_video(video_id) do - %Media.Video{state: :crf_searching} = video -> - reply_for_active_work(worker_id, socket, video, request_mode) + handle_active_video_work_request(worker_id, socket, video_id, request_mode) + end + end - %Media.Video{} -> - socket = clear_assigned_video(worker_id, socket) - resume_or_claim_work(worker_id, socket, request_mode) + defp handle_active_video_work_request(worker_id, socket, video_id, request_mode) do + case Media.get_video(video_id) do + %Media.Video{state: :crf_searching} = video -> + request_mode = + case WorkerSessions.get(worker_id) do + %{active_video_id: ^video_id} = session -> + maybe_resume_mode(session, request_mode) - nil -> - {:reply, {:error, WorkerProtocol.error(:unknown_worker_session)}, socket} - end + _ -> + request_mode + end + + reply_for_active_work(worker_id, socket, video, request_mode) + + %Media.Video{} -> + socket = clear_assigned_video(worker_id, socket) + resume_or_claim_work(worker_id, socket, request_mode) + + nil -> + {:reply, {:error, WorkerProtocol.error(:unknown_worker_session)}, socket} end end diff --git a/test/reencodarr_web/channels/worker_channel_test.exs b/test/reencodarr_web/channels/worker_channel_test.exs index ae8b0738..8c5b59aa 100644 --- a/test/reencodarr_web/channels/worker_channel_test.exs +++ b/test/reencodarr_web/channels/worker_channel_test.exs @@ -378,6 +378,17 @@ defmodule ReencodarrWeb.WorkerChannelTest do :ok, %{accepted: true, event: "crf_search_progress"} + assert {:ok, _} = + WorkerSessions.set_transfer_progress(server_worker_id, %{ + job_id: Integer.to_string(video_id), + video_id: video_id, + transfer_id: Integer.to_string(video_id), + filename: Path.basename(video.path), + percent: 100.0, + bytes_sent: 8, + total_bytes: 8 + }) + assert_reply push(socket, "pull_work", %{}), :ok, %{ @@ -407,32 +418,35 @@ defmodule ReencodarrWeb.WorkerChannelTest do token = "test-worker-token" Application.put_env(:reencodarr, :worker_token, token) - {:ok, video} = - Fixtures.video_fixture(%{ - state: :crf_searching, - crf_search_worker_id: "worker-a" - }) + with_temp_file("abcdefgh", ".mkv", fn path -> + {:ok, video} = + Fixtures.video_fixture(%{ + path: path, + size: 8, + state: :crf_searching, + crf_search_worker_id: "worker-a" + }) - video_id = video.id + video_id = video.id - assert {:ok, socket} = connect(WorkerSocket, %{"token" => token}) - assert {:ok, _join_payload, socket} = subscribe_and_join(socket, "workers:crf_search") + assert {:ok, socket} = connect(WorkerSocket, %{"token" => token}) + assert {:ok, _join_payload, socket} = subscribe_and_join(socket, "workers:crf_search") - assert_reply push(socket, "announce", announce_payload(worker_id: "worker-a")), - :ok, - %{accepted: true, protocol_version: 1} + assert_reply push(socket, "announce", announce_payload(worker_id: "worker-a")), + :ok, + %{accepted: true, protocol_version: 1} - assert_reply push(socket, "pull_work", %{}), - :ok, - %{ - status: "job_in_progress", - video_id: ^video_id, - crf_search_args: crf_search_args - } + assert_reply push(socket, "pull_work", %{}), + :ok, + %{ + status: "job_assigned", + video_id: ^video_id, + crf_search_args: crf_search_args + } - assert "crf-search" in crf_search_args - refute_push "transfer_started", _, 50 - refute_push "transfer_chunk", _, 50 + assert "crf-search" in crf_search_args + assert_push "transfer_started", %{status: "transfer_started", video_id: ^video_id} + end) after Application.delete_env(:reencodarr, :worker_token) end @@ -470,7 +484,7 @@ defmodule ReencodarrWeb.WorkerChannelTest do assert_reply push(socket, "pull_work", %{}), :ok, %{ - status: "job_in_progress", + status: "job_assigned", video_id: ^video_id, crf_search_args: in_progress_args } @@ -574,21 +588,16 @@ defmodule ReencodarrWeb.WorkerChannelTest do :ok, %{accepted: true, protocol_version: 1} - pull_work_reply = assert_reply push(socket, "pull_work", %{}), :ok, _ - - assert %{ - video_id: ^video_id, - crf_search_args: crf_search_args - } = pull_work_reply.payload - - case pull_work_reply.payload do - %{status: "job_assigned", chunk_size_bytes: _chunk_size_bytes} -> - assert "crf-search" in crf_search_args - assert_push "transfer_started", %{status: "transfer_started", video_id: ^video_id} + assert_reply push(socket, "pull_work", %{}), + :ok, + %{ + status: "job_assigned", + video_id: ^video_id, + crf_search_args: crf_search_args + } - %{status: "job_in_progress"} -> - :ok - end + assert "crf-search" in crf_search_args + assert_push "transfer_started", %{status: "transfer_started", video_id: ^video_id} end) after Application.delete_env(:reencodarr, :worker_token) From 7db89c011553b76da86ad9fd3ec731b5b59941a3 Mon Sep 17 00:00:00 2001 From: Mika Cohen Date: Wed, 8 Jul 2026 18:55:22 -0600 Subject: [PATCH 54/70] touch worker sessions on progress and pull work --- lib/reencodarr/ab_av1/worker_sessions.ex | 12 ++++- lib/reencodarr_web/channels/worker_channel.ex | 2 + .../ab_av1/worker_sessions_test.exs | 47 +++++++++++++++++++ .../channels/worker_channel_test.exs | 33 +++++++++++++ 4 files changed, 92 insertions(+), 2 deletions(-) diff --git a/lib/reencodarr/ab_av1/worker_sessions.ex b/lib/reencodarr/ab_av1/worker_sessions.ex index e82512ab..6422cd86 100644 --- a/lib/reencodarr/ab_av1/worker_sessions.ex +++ b/lib/reencodarr/ab_av1/worker_sessions.ex @@ -142,15 +142,23 @@ defmodule Reencodarr.AbAv1.WorkerSessions do update_session_reply(server_worker_id, state, fn session -> active_video_id = Map.get(progress, :video_id, session.active_video_id) - %{session | active_video_id: active_video_id, transfer_progress: progress} + now = now() + + %{ + session + | active_video_id: active_video_id, + transfer_progress: progress, + last_seen_at: now + } end) end def handle_call({:set_crf_search_progress, server_worker_id, progress}, _from, state) do update_session_reply(server_worker_id, state, fn session -> progress = merge_crf_search_progress(session.crf_search_progress, progress) + now = now() - %{session | crf_search_progress: progress} + %{session | crf_search_progress: progress, last_seen_at: now} end) end diff --git a/lib/reencodarr_web/channels/worker_channel.ex b/lib/reencodarr_web/channels/worker_channel.ex index d37f3884..1677f847 100644 --- a/lib/reencodarr_web/channels/worker_channel.ex +++ b/lib/reencodarr_web/channels/worker_channel.ex @@ -133,6 +133,8 @@ defmodule ReencodarrWeb.WorkerChannel do end defp handle_work_request(payload, %{assigns: %{worker_id: worker_id}} = socket) do + _ = WorkerSessions.touch(worker_id) + request_mode = work_request_mode(payload) case socket.assigns[:current_video_id] do diff --git a/test/reencodarr/ab_av1/worker_sessions_test.exs b/test/reencodarr/ab_av1/worker_sessions_test.exs index e4d76848..15058301 100644 --- a/test/reencodarr/ab_av1/worker_sessions_test.exs +++ b/test/reencodarr/ab_av1/worker_sessions_test.exs @@ -136,6 +136,53 @@ defmodule Reencodarr.AbAv1.WorkerSessionsTest do assert session.crf_search_progress.total_samples == 5 end + test "touches session when transfer progress is reported" do + assert {:ok, _session} = WorkerSessions.register(worker_session_attrs()) + + initial_session = WorkerSessions.get("worker-server-1") + initial_last_seen = initial_session.last_seen_at + + Process.sleep(1_100) + + assert {:ok, session} = + WorkerSessions.set_transfer_progress("worker-server-1", %{ + job_id: "job-1", + video_id: 123, + transfer_id: "job-1", + filename: "sample.mkv", + percent: 25.0, + bytes_sent: 2_621_440, + total_bytes: 10_485_760, + bytes_per_second: 1_048_576, + eta: 15, + chunk_index: 2, + total_chunks: 8 + }) + + assert DateTime.compare(session.last_seen_at, initial_last_seen) == :gt + end + + test "touches session when CRF search progress is reported" do + assert {:ok, _session} = WorkerSessions.register(worker_session_attrs()) + + initial_session = WorkerSessions.get("worker-server-1") + initial_last_seen = initial_session.last_seen_at + + Process.sleep(1_100) + + assert {:ok, session} = + WorkerSessions.set_crf_search_progress("worker-server-1", %{ + video_id: 123, + percent: 25.0, + fps: 24.0, + crf: 28.0, + sample_num: 3, + total_samples: 5 + }) + + assert DateTime.compare(session.last_seen_at, initial_last_seen) == :gt + end + test "replaces reconnecting client sessions without dropping active state" do {:ok, video} = Fixtures.video_fixture(%{state: :crf_searching}) diff --git a/test/reencodarr_web/channels/worker_channel_test.exs b/test/reencodarr_web/channels/worker_channel_test.exs index 8c5b59aa..659ea579 100644 --- a/test/reencodarr_web/channels/worker_channel_test.exs +++ b/test/reencodarr_web/channels/worker_channel_test.exs @@ -215,6 +215,39 @@ defmodule ReencodarrWeb.WorkerChannelTest do Application.delete_env(:reencodarr, :worker_token) end + test "tracks last-seen for pull_work requests on active sessions" do + token = "test-worker-token" + Application.put_env(:reencodarr, :worker_token, token) + + {:ok, video} = Fixtures.video_fixture(%{state: :analyzed}) + video_id = video.id + + assert {:ok, socket} = connect(WorkerSocket, %{"token" => token}) + assert {:ok, _join_payload, socket} = subscribe_and_join(socket, "workers:crf_search") + + assert_reply push(socket, "announce", announce_payload(worker_id: "worker-a")), :ok, %{ + accepted: true + } + + assert_reply push(socket, "pull_work", %{}), :ok, %{ + status: "job_assigned", + video_id: ^video_id + } + + [session] = WorkerSessions.list() + first_seen_at = session.last_seen_at + + Process.sleep(1_100) + + assert_reply push(socket, "pull_work", %{}), :ok, %{status: status, video_id: ^video_id} + assert status in ["job_assigned", "job_in_progress"] + + [updated_session] = WorkerSessions.list() + assert DateTime.compare(updated_session.last_seen_at, first_seen_at) == :gt + after + Application.delete_env(:reencodarr, :worker_token) + end + test "broadcasts transfer and CRF progress updates" do token = "test-worker-token" Application.put_env(:reencodarr, :worker_token, token) From 5cfd1ec892ec298682a492116bf31d8295a2a411 Mon Sep 17 00:00:00 2001 From: Mika Cohen Date: Wed, 8 Jul 2026 19:17:05 -0600 Subject: [PATCH 55/70] add explicit worker session phase --- lib/reencodarr/ab_av1/worker_sessions.ex | 63 ++++++++++-- lib/reencodarr_web/channels/worker_channel.ex | 9 +- lib/reencodarr_web/live/workers_live.ex | 96 +++++++++++-------- .../ab_av1/worker_sessions_test.exs | 34 +++++++ .../channels/worker_channel_test.exs | 10 +- .../reencodarr_web/live/workers_live_test.exs | 43 ++++++++- 6 files changed, 200 insertions(+), 55 deletions(-) diff --git a/lib/reencodarr/ab_av1/worker_sessions.ex b/lib/reencodarr/ab_av1/worker_sessions.ex index 6422cd86..0b76bb29 100644 --- a/lib/reencodarr/ab_av1/worker_sessions.ex +++ b/lib/reencodarr/ab_av1/worker_sessions.ex @@ -18,6 +18,7 @@ defmodule Reencodarr.AbAv1.WorkerSessions do version: String.t(), protocol_version: pos_integer(), capabilities: map(), + phase: :idle | :receiving_input | :crf_searching, active_video_id: integer() | nil, transfer_progress: map() | nil, crf_search_progress: map() | nil, @@ -42,8 +43,9 @@ defmodule Reencodarr.AbAv1.WorkerSessions do GenServer.call(__MODULE__, {:unregister, server_worker_id}) end - def assign_video(server_worker_id, video_id) when is_integer(video_id) do - GenServer.call(__MODULE__, {:assign_video, server_worker_id, video_id}) + def assign_video(server_worker_id, video_id, phase \\ :crf_searching) + when is_integer(video_id) and phase in [:receiving_input, :crf_searching] do + GenServer.call(__MODULE__, {:assign_video, server_worker_id, video_id, phase}) end def clear_video(server_worker_id) do @@ -54,6 +56,10 @@ defmodule Reencodarr.AbAv1.WorkerSessions do GenServer.call(__MODULE__, {:set_transfer_progress, server_worker_id, progress}) end + def finish_transfer(server_worker_id) do + GenServer.call(__MODULE__, {:finish_transfer, server_worker_id}) + end + def set_crf_search_progress(server_worker_id, progress) when is_map(progress) do GenServer.call(__MODULE__, {:set_crf_search_progress, server_worker_id, progress}) end @@ -126,15 +132,27 @@ defmodule Reencodarr.AbAv1.WorkerSessions do {:reply, :ok, state} end - def handle_call({:assign_video, server_worker_id, video_id}, _from, state) do + def handle_call({:assign_video, server_worker_id, video_id, phase}, _from, state) do update_session_reply(server_worker_id, state, fn session -> - %{session | active_video_id: video_id, transfer_progress: nil, crf_search_progress: nil} + %{ + session + | phase: phase, + active_video_id: video_id, + transfer_progress: nil, + crf_search_progress: nil + } end) end def handle_call({:clear_video, server_worker_id}, _from, state) do update_session_reply(server_worker_id, state, fn session -> - %{session | active_video_id: nil, transfer_progress: nil, crf_search_progress: nil} + %{ + session + | phase: :idle, + active_video_id: nil, + transfer_progress: nil, + crf_search_progress: nil + } end) end @@ -146,19 +164,41 @@ defmodule Reencodarr.AbAv1.WorkerSessions do %{ session - | active_video_id: active_video_id, + | phase: :receiving_input, + active_video_id: active_video_id, transfer_progress: progress, + crf_search_progress: nil, last_seen_at: now } end) end + def handle_call({:finish_transfer, server_worker_id}, _from, state) do + update_session_reply(server_worker_id, state, fn session -> + phase = + case session.active_video_id do + nil -> :idle + _video_id -> :crf_searching + end + + %{session | phase: phase, transfer_progress: nil, last_seen_at: now()} + end) + end + def handle_call({:set_crf_search_progress, server_worker_id, progress}, _from, state) do update_session_reply(server_worker_id, state, fn session -> progress = merge_crf_search_progress(session.crf_search_progress, progress) + active_video_id = Map.get(progress, :video_id, session.active_video_id) now = now() - %{session | crf_search_progress: progress, last_seen_at: now} + %{ + session + | phase: :crf_searching, + active_video_id: active_video_id, + transfer_progress: nil, + crf_search_progress: progress, + last_seen_at: now + } end) end @@ -243,6 +283,7 @@ defmodule Reencodarr.AbAv1.WorkerSessions do version: version, protocol_version: protocol_version, capabilities: capabilities, + phase: :idle, active_video_id: nil, transfer_progress: nil, crf_search_progress: nil, @@ -339,6 +380,9 @@ defmodule Reencodarr.AbAv1.WorkerSessions do {:ok, existing_session} -> active_video_id = resumable_active_video_id(existing_session.active_video_id) + phase = + if(active_video_id, do: Map.get(existing_session, :phase, :crf_searching), else: :idle) + session = build_session( server_worker_id, @@ -350,9 +394,12 @@ defmodule Reencodarr.AbAv1.WorkerSessions do ) |> Map.put(:connected_at, existing_session.connected_at) |> Map.put(:active_video_id, active_video_id) + |> Map.put(:phase, phase) |> Map.put( :transfer_progress, - if(active_video_id, do: existing_session.transfer_progress) + if(active_video_id && phase == :receiving_input, + do: existing_session.transfer_progress + ) ) |> Map.put( :crf_search_progress, diff --git a/lib/reencodarr_web/channels/worker_channel.ex b/lib/reencodarr_web/channels/worker_channel.ex index 1677f847..039abbee 100644 --- a/lib/reencodarr_web/channels/worker_channel.ex +++ b/lib/reencodarr_web/channels/worker_channel.ex @@ -233,7 +233,7 @@ defmodule ReencodarrWeb.WorkerChannel do defp attach_announced_work(socket, client_worker_id, _session) do case Media.get_worker_crf_searching_video(client_worker_id) do %Media.Video{} = video -> - _ = WorkerSessions.assign_video(socket.assigns.worker_id, video.id) + _ = WorkerSessions.assign_video(socket.assigns.worker_id, video.id, :crf_searching) attach_active_video(socket, video.id) nil -> @@ -348,7 +348,7 @@ defmodule ReencodarrWeb.WorkerChannel do target_vmaf = Reencodarr.Rules.vmaf_target(video) with {:ok, _video} <- Media.mark_as_worker_crf_searching(video, worker_dispatch_id(socket)), - {:ok, _session} <- WorkerSessions.assign_video(worker_id, video.id) do + {:ok, _session} <- WorkerSessions.assign_video(worker_id, video.id, :receiving_input) do socket = prepare_transfer(socket, video, target_vmaf) {:reply, {:ok, WorkerProtocol.work_assigned(video, target_vmaf)}, socket} else @@ -361,7 +361,7 @@ defmodule ReencodarrWeb.WorkerChannel do defp reply_for_active_work(worker_id, socket, video, :resend_input) do target_vmaf = socket.assigns[:current_vmaf_target] || Reencodarr.Rules.vmaf_target(video) - case WorkerSessions.assign_video(worker_id, video.id) do + case WorkerSessions.assign_video(worker_id, video.id, :receiving_input) do {:ok, _session} -> socket = prepare_transfer(socket, video, target_vmaf) {:reply, {:ok, WorkerProtocol.work_assigned(video, target_vmaf)}, socket} @@ -614,7 +614,7 @@ defmodule ReencodarrWeb.WorkerChannel do when is_binary(dispatch_id) <- Media.get_video(video_id), true <- dispatch_id == worker_dispatch_id(socket), false <- assigned_to_other_worker?(worker_id, video_id), - {:ok, _session} <- WorkerSessions.assign_video(worker_id, video_id) do + {:ok, _session} <- WorkerSessions.assign_video(worker_id, video_id, :crf_searching) do {:ok, socket |> assign(:current_video_id, video_id) @@ -846,6 +846,7 @@ defmodule ReencodarrWeb.WorkerChannel do :eof -> close_transfer_stream(io_device) + _ = WorkerSessions.finish_transfer(socket.assigns.worker_id) push( socket, diff --git a/lib/reencodarr_web/live/workers_live.ex b/lib/reencodarr_web/live/workers_live.ex index ce202f7d..9c2a5848 100644 --- a/lib/reencodarr_web/live/workers_live.ex +++ b/lib/reencodarr_web/live/workers_live.ex @@ -116,22 +116,25 @@ defmodule ReencodarrWeb.WorkersLive do
- <%= if show_crf_panel?(worker) do %> - <.crf_search_panel - video={worker_crf_video(worker)} - results={worker_crf_results(worker)} - sample={worker_crf_sample(worker)} - status={worker_crf_status(worker)} - show_controls={true} - show_queue={false} - show_empty_chart={true} - suspend_event="pause_worker_crf_search" - resume_event="resume_worker_crf_search" - fail_event="stop_worker_crf_search" - worker_id={worker.server_worker_id} - /> - <% else %> - <.transfer_panel worker={worker} video={active_video(worker)} /> + <%= case worker_phase(worker) do %> + <% :receiving_input -> %> + <.transfer_panel worker={worker} video={active_video(worker)} /> + <% :crf_searching -> %> + <.crf_search_panel + video={worker_crf_video(worker)} + results={worker_crf_results(worker)} + sample={worker_crf_sample(worker)} + status={worker_crf_status(worker)} + show_controls={true} + show_queue={false} + show_empty_chart={true} + suspend_event="pause_worker_crf_search" + resume_event="resume_worker_crf_search" + fail_event="stop_worker_crf_search" + worker_id={worker.server_worker_id} + /> + <% :idle -> %> + <.idle_panel /> <% end %>
@@ -143,6 +146,15 @@ defmodule ReencodarrWeb.WorkersLive do """ end + defp idle_panel(assigns) do + ~H""" +
+

Idle

+
Waiting for work.
+
+ """ + end + defp control_worker(socket, worker_id, action, message) do Phoenix.PubSub.broadcast( Reencodarr.PubSub, @@ -201,35 +213,37 @@ defmodule ReencodarrWeb.WorkersLive do """ end - defp worker_status(%{active_video_id: nil}), do: "Idle" - - defp worker_status(%{transfer_progress: progress}) when not is_nil(progress), - do: "Receiving input" - - defp worker_status(%{crf_search_progress: progress}) when not is_nil(progress), - do: "CRF search" + defp worker_status(worker) do + case worker_phase(worker) do + :idle -> "Idle" + :receiving_input -> "Receiving input" + :crf_searching -> crf_search_status(worker) + end + end - defp worker_status(%{active_video_id: video_id}) do + defp crf_search_status(%{active_video_id: video_id}) when is_integer(video_id) do case Media.get_video(video_id) do %Media.Video{state: state} -> Atom.to_string(state) nil -> "missing" end end - defp worker_crf_status(%{active_video_id: nil}), do: :idle + defp crf_search_status(_worker), do: "CRF search" + + defp worker_crf_status(%{phase: :crf_searching}), do: :processing - defp worker_crf_status(%{transfer_progress: progress}) when not is_nil(progress), - do: :idle + defp worker_crf_status(_worker), do: :idle - defp worker_crf_status(%{crf_search_progress: progress}) when not is_nil(progress), - do: :processing + defp worker_phase(%{phase: phase}) when phase in [:idle, :receiving_input, :crf_searching], + do: phase - defp worker_crf_status(%{active_video_id: _video_id}), do: :processing + defp worker_phase(%{active_video_id: nil}), do: :idle - defp show_crf_panel?(%{transfer_progress: progress}) when not is_nil(progress), do: false - defp show_crf_panel?(%{crf_search_progress: progress}) when not is_nil(progress), do: true - defp show_crf_panel?(%{active_video_id: video_id}) when is_integer(video_id), do: true - defp show_crf_panel?(_worker), do: false + defp worker_phase(%{transfer_progress: progress}) when not is_nil(progress), + do: :receiving_input + + defp worker_phase(%{active_video_id: video_id}) when is_integer(video_id), do: :crf_searching + defp worker_phase(_worker), do: :idle defp worker_crf_video(worker) do case active_video(worker) do @@ -380,11 +394,13 @@ defmodule ReencodarrWeb.WorkersLive do defp format_chunk_progress(_), do: "-" - defp status_badge_class(%{active_video_id: nil}), - do: - "inline-flex rounded-full border border-emerald-900 bg-emerald-950 px-2 py-1 text-xs font-medium text-emerald-300" + defp status_badge_class(worker) do + case worker_phase(worker) do + :idle -> + "inline-flex rounded-full border border-emerald-900 bg-emerald-950 px-2 py-1 text-xs font-medium text-emerald-300" - defp status_badge_class(%{active_video_id: _video_id}), - do: - "inline-flex rounded-full border border-cyan-900 bg-cyan-950 px-2 py-1 text-xs font-medium text-cyan-300" + _active -> + "inline-flex rounded-full border border-cyan-900 bg-cyan-950 px-2 py-1 text-xs font-medium text-cyan-300" + end + end end diff --git a/test/reencodarr/ab_av1/worker_sessions_test.exs b/test/reencodarr/ab_av1/worker_sessions_test.exs index 15058301..544db0c0 100644 --- a/test/reencodarr/ab_av1/worker_sessions_test.exs +++ b/test/reencodarr/ab_av1/worker_sessions_test.exs @@ -23,6 +23,7 @@ defmodule Reencodarr.AbAv1.WorkerSessionsTest do assert {:error, :unknown_worker_session} = WorkerSessions.touch("missing-worker") assert {:error, :unknown_worker_session} = WorkerSessions.assign_video("missing-worker", 123) assert {:error, :unknown_worker_session} = WorkerSessions.clear_video("missing-worker") + assert {:error, :unknown_worker_session} = WorkerSessions.finish_transfer("missing-worker") end test "expires stale worker sessions" do @@ -97,9 +98,39 @@ defmodule Reencodarr.AbAv1.WorkerSessionsTest do assert {:ok, session} = WorkerSessions.assign_video("worker-server-1", 123) assert session.active_video_id == 123 + assert session.phase == :crf_searching [listed_session] = WorkerSessions.list() assert listed_session.active_video_id == 123 + assert listed_session.phase == :crf_searching + end + + test "moves from receiving input to CRF search without clearing the active video" do + assert {:ok, _session} = WorkerSessions.register(worker_session_attrs()) + + assert {:ok, session} = WorkerSessions.assign_video("worker-server-1", 123, :receiving_input) + assert session.active_video_id == 123 + assert session.phase == :receiving_input + + assert {:ok, session} = + WorkerSessions.set_transfer_progress("worker-server-1", %{ + job_id: "job-1", + video_id: 123, + transfer_id: "job-1", + filename: "sample.mkv", + percent: 100.0, + bytes_sent: 10_485_760, + total_bytes: 10_485_760 + }) + + assert session.active_video_id == 123 + assert session.phase == :receiving_input + assert session.transfer_progress.percent == 100.0 + + assert {:ok, session} = WorkerSessions.finish_transfer("worker-server-1") + assert session.active_video_id == 123 + assert session.phase == :crf_searching + assert is_nil(session.transfer_progress) end test "keeps CRF sample metadata when later progress omits it" do @@ -207,6 +238,7 @@ defmodule Reencodarr.AbAv1.WorkerSessionsTest do }) assert session.active_video_id == video.id + assert session.phase == :crf_searching assert {:ok, session} = WorkerSessions.register( @@ -219,6 +251,7 @@ defmodule Reencodarr.AbAv1.WorkerSessionsTest do assert session.server_worker_id == "worker-server-2" assert session.client_worker_id == "worker-client-1" assert session.active_video_id == video.id + assert session.phase == :crf_searching assert session.crf_search_progress.video_id == video.id assert session.crf_search_progress.sample_num == 2 assert is_nil(WorkerSessions.get("worker-server-1")) @@ -255,6 +288,7 @@ defmodule Reencodarr.AbAv1.WorkerSessionsTest do ) assert is_nil(session.active_video_id) + assert session.phase == :idle assert is_nil(session.crf_search_progress) assert Media.get_video(video.id).state == :crf_searched end diff --git a/test/reencodarr_web/channels/worker_channel_test.exs b/test/reencodarr_web/channels/worker_channel_test.exs index 659ea579..484a93c5 100644 --- a/test/reencodarr_web/channels/worker_channel_test.exs +++ b/test/reencodarr_web/channels/worker_channel_test.exs @@ -327,8 +327,8 @@ defmodule ReencodarrWeb.WorkerChannelTest do assert filename == Path.basename(video.path) session = WorkerSessions.get(server_worker_id) - assert session.transfer_progress - assert session.transfer_progress.percent == 25.5 + assert session.phase == :crf_searching + assert is_nil(session.transfer_progress) assert session.crf_search_progress.percent == 62.0 Phoenix.PubSub.broadcast( @@ -695,6 +695,7 @@ defmodule ReencodarrWeb.WorkerChannelTest do session = WorkerSessions.get(socket.assigns.worker_id) assert session.active_video_id == assigned_video_id + assert session.phase == :receiving_input assert session.transfer_progress.video_id == assigned_video_id assert session.transfer_progress.percent == 0.0 assert session.transfer_progress.bytes_sent == 0 @@ -737,6 +738,11 @@ defmodule ReencodarrWeb.WorkerChannelTest do total_bytes: ^content_size, total_chunks: 2 } + + session = WorkerSessions.get(socket.assigns.worker_id) + assert session.active_video_id == assigned_video_id + assert session.phase == :crf_searching + assert is_nil(session.transfer_progress) end) after Application.delete_env(:reencodarr, :worker_token) diff --git a/test/reencodarr_web/live/workers_live_test.exs b/test/reencodarr_web/live/workers_live_test.exs index 4566be2b..8581ef87 100644 --- a/test/reencodarr_web/live/workers_live_test.exs +++ b/test/reencodarr_web/live/workers_live_test.exs @@ -34,8 +34,10 @@ defmodule ReencodarrWeb.WorkersLiveTest do assert {:ok, view, html} = live(conn, ~p"/workers") assert html =~ "worker-client-1" assert html =~ "Idle" + refute html =~ "Receiving Input" - assert {:ok, _session} = WorkerSessions.assign_video("worker-server-1", video.id) + assert {:ok, _session} = + WorkerSessions.assign_video("worker-server-1", video.id, :receiving_input) assert {:ok, _session} = WorkerSessions.touch("worker-server-1", %{ @@ -132,4 +134,43 @@ defmodule ReencodarrWeb.WorkersLiveTest do assert html =~ "CRF 28.0 -> 95.4 VMAF" refute html =~ "Receiving Input" end + + test "keeps the crf panel active after input transfer finishes", %{conn: conn} do + {:ok, _session} = + WorkerSessions.register(%{ + server_worker_id: "worker-server-3", + client_worker_id: "worker-client-3", + protocol_version: 1, + version: "0.10.0", + capabilities: %{"crf_search" => true} + }) + + {:ok, video} = Fixtures.video_fixture(%{state: :crf_searching}) + + assert {:ok, _session} = + WorkerSessions.assign_video("worker-server-3", video.id, :receiving_input) + + assert {:ok, _session} = + WorkerSessions.set_transfer_progress("worker-server-3", %{ + job_id: Integer.to_string(video.id), + video_id: video.id, + transfer_id: Integer.to_string(video.id), + filename: Path.basename(video.path), + percent: 100.0, + bytes_sent: video.size, + total_bytes: video.size + }) + + assert {:ok, _session} = WorkerSessions.finish_transfer("worker-server-3") + + {:ok, view, _html} = live(conn, ~p"/workers") + send(view.pid, {:worker_sessions_updated, %{sessions: WorkerSessions.list()}}) + + html = render(view) + assert html =~ "worker-client-3" + assert html =~ "crf_searching" + assert html =~ "CRF Search" + refute html =~ "Receiving Input" + refute html =~ "-%" + end end From 72ee69e33f5dbd18689aedf79dc8331fca599020 Mon Sep 17 00:00:00 2001 From: Mika Cohen Date: Wed, 8 Jul 2026 19:40:29 -0600 Subject: [PATCH 56/70] use state machines for worker job completion --- .../ab_av1/worker_job_state_machine.ex | 85 ++++++++++++ lib/reencodarr/ab_av1/worker_sessions.ex | 70 ++++------ lib/reencodarr_web/channels/worker_channel.ex | 130 +++++++++++------- .../ab_av1/worker_job_state_machine_test.exs | 80 +++++++++++ .../channels/worker_channel_test.exs | 12 +- 5 files changed, 278 insertions(+), 99 deletions(-) create mode 100644 lib/reencodarr/ab_av1/worker_job_state_machine.ex create mode 100644 test/reencodarr/ab_av1/worker_job_state_machine_test.exs diff --git a/lib/reencodarr/ab_av1/worker_job_state_machine.ex b/lib/reencodarr/ab_av1/worker_job_state_machine.ex new file mode 100644 index 00000000..098a8240 --- /dev/null +++ b/lib/reencodarr/ab_av1/worker_job_state_machine.ex @@ -0,0 +1,85 @@ +defmodule Reencodarr.AbAv1.WorkerJobStateMachine do + @moduledoc """ + State machine for one connected worker's current job phase. + """ + + require Logger + + @type phase :: :idle | :receiving_input | :crf_searching + + @valid_phases [:idle, :receiving_input, :crf_searching] + + @valid_transitions %{ + idle: [:receiving_input, :crf_searching], + receiving_input: [:receiving_input, :crf_searching, :idle], + crf_searching: [:crf_searching, :receiving_input, :idle] + } + + @spec valid_phases() :: [phase()] + def valid_phases, do: @valid_phases + + @spec valid_transition?(phase(), phase()) :: boolean() + def valid_transition?(from_phase, to_phase) + when from_phase in @valid_phases and to_phase in @valid_phases do + to_phase == from_phase or to_phase in Map.fetch!(@valid_transitions, from_phase) + end + + def valid_transition?(_from_phase, _to_phase), do: false + + def transition(session, to_phase, attrs \\ %{}) + + @spec transition(map(), phase(), map()) :: {:ok, map()} | {:error, :invalid_worker_phase} + def transition(session, to_phase, attrs) when to_phase in @valid_phases do + from_phase = Map.get(session, :phase, :idle) + + if valid_transition?(from_phase, to_phase) do + {:ok, session |> Map.merge(attrs) |> Map.put(:phase, to_phase)} + else + Logger.warning("Invalid worker job transition from #{from_phase} to #{to_phase}") + {:error, :invalid_worker_phase} + end + end + + def transition(_session, _to_phase, _attrs), do: {:error, :invalid_worker_phase} + + @spec assign_video(map(), integer(), phase()) :: {:ok, map()} | {:error, :invalid_worker_phase} + def assign_video(session, video_id, phase) when is_integer(video_id) do + transition(session, phase, %{ + active_video_id: video_id, + transfer_progress: nil, + crf_search_progress: nil + }) + end + + @spec clear_video(map()) :: {:ok, map()} + def clear_video(session) do + {:ok, + %{session | active_video_id: nil, transfer_progress: nil, crf_search_progress: nil} + |> Map.put(:phase, :idle)} + end + + @spec set_transfer_progress(map(), map()) :: {:ok, map()} | {:error, :invalid_worker_phase} + def set_transfer_progress(session, progress) do + transition(session, :receiving_input, %{ + active_video_id: Map.get(progress, :video_id, session.active_video_id), + transfer_progress: progress, + crf_search_progress: nil + }) + end + + @spec finish_transfer(map()) :: {:ok, map()} + def finish_transfer(%{active_video_id: nil} = session), do: clear_video(session) + + def finish_transfer(session) do + {:ok, %{session | phase: :crf_searching, transfer_progress: nil}} + end + + @spec set_crf_search_progress(map(), map()) :: {:ok, map()} | {:error, :invalid_worker_phase} + def set_crf_search_progress(session, progress) do + transition(session, :crf_searching, %{ + active_video_id: Map.get(progress, :video_id, session.active_video_id), + transfer_progress: nil, + crf_search_progress: progress + }) + end +end diff --git a/lib/reencodarr/ab_av1/worker_sessions.ex b/lib/reencodarr/ab_av1/worker_sessions.ex index 0b76bb29..98e65eca 100644 --- a/lib/reencodarr/ab_av1/worker_sessions.ex +++ b/lib/reencodarr/ab_av1/worker_sessions.ex @@ -5,6 +5,7 @@ defmodule Reencodarr.AbAv1.WorkerSessions do use GenServer + alias Reencodarr.AbAv1.WorkerJobStateMachine alias Reencodarr.Dashboard.Events alias Reencodarr.Media alias Reencodarr.Media.VideoStateMachine @@ -134,71 +135,39 @@ defmodule Reencodarr.AbAv1.WorkerSessions do def handle_call({:assign_video, server_worker_id, video_id, phase}, _from, state) do update_session_reply(server_worker_id, state, fn session -> - %{ - session - | phase: phase, - active_video_id: video_id, - transfer_progress: nil, - crf_search_progress: nil - } + WorkerJobStateMachine.assign_video(session, video_id, phase) end) end def handle_call({:clear_video, server_worker_id}, _from, state) do update_session_reply(server_worker_id, state, fn session -> - %{ - session - | phase: :idle, - active_video_id: nil, - transfer_progress: nil, - crf_search_progress: nil - } + WorkerJobStateMachine.clear_video(session) end) end def handle_call({:set_transfer_progress, server_worker_id, progress}, _from, state) do update_session_reply(server_worker_id, state, fn session -> - active_video_id = Map.get(progress, :video_id, session.active_video_id) - - now = now() - - %{ - session - | phase: :receiving_input, - active_video_id: active_video_id, - transfer_progress: progress, - crf_search_progress: nil, - last_seen_at: now - } + with {:ok, session} <- WorkerJobStateMachine.set_transfer_progress(session, progress) do + {:ok, %{session | last_seen_at: now()}} + end end) end def handle_call({:finish_transfer, server_worker_id}, _from, state) do update_session_reply(server_worker_id, state, fn session -> - phase = - case session.active_video_id do - nil -> :idle - _video_id -> :crf_searching - end - - %{session | phase: phase, transfer_progress: nil, last_seen_at: now()} + with {:ok, session} <- WorkerJobStateMachine.finish_transfer(session) do + {:ok, %{session | last_seen_at: now()}} + end end) end def handle_call({:set_crf_search_progress, server_worker_id, progress}, _from, state) do update_session_reply(server_worker_id, state, fn session -> progress = merge_crf_search_progress(session.crf_search_progress, progress) - active_video_id = Map.get(progress, :video_id, session.active_video_id) - now = now() - %{ - session - | phase: :crf_searching, - active_video_id: active_video_id, - transfer_progress: nil, - crf_search_progress: progress, - last_seen_at: now - } + with {:ok, session} <- WorkerJobStateMachine.set_crf_search_progress(session, progress) do + {:ok, %{session | last_seen_at: now()}} + end end) end @@ -421,9 +390,18 @@ defmodule Reencodarr.AbAv1.WorkerSessions do defp update_session_reply(server_worker_id, state, update_fun) do case lookup_session(server_worker_id) do {:ok, session} -> - updated_session = update_fun.(session) - :ok = put_session(updated_session) - {:reply, {:ok, updated_session}, state} + case update_fun.(session) do + {:ok, updated_session} -> + :ok = put_session(updated_session) + {:reply, {:ok, updated_session}, state} + + {:error, reason} -> + {:reply, {:error, reason}, state} + + updated_session -> + :ok = put_session(updated_session) + {:reply, {:ok, updated_session}, state} + end :error -> {:reply, {:error, :unknown_worker_session}, state} diff --git a/lib/reencodarr_web/channels/worker_channel.ex b/lib/reencodarr_web/channels/worker_channel.ex index 039abbee..e43a71dc 100644 --- a/lib/reencodarr_web/channels/worker_channel.ex +++ b/lib/reencodarr_web/channels/worker_channel.ex @@ -497,22 +497,25 @@ defmodule ReencodarrWeb.WorkerChannel do clear_assigned_video(worker_id, socket)} video -> - socket = - apply_completion_result( - worker_id, - socket, - video, - completion.result, - completion.chosen_crf - ) + case apply_completion_result( + worker_id, + socket, + video, + completion.result, + completion.chosen_crf + ) do + {:ok, socket} -> + Events.broadcast_event(:crf_search_completed, %{ + video_id: completion.video_id, + result: completion.result, + chosen_crf: completion.chosen_crf + }) - Events.broadcast_event(:crf_search_completed, %{ - video_id: completion.video_id, - result: completion.result, - chosen_crf: completion.chosen_crf - }) + {:reply, {:ok, WorkerProtocol.event_ack("crf_search_completed")}, socket} - {:reply, {:ok, WorkerProtocol.event_ack("crf_search_completed")}, socket} + {:error, reason, socket} -> + {:reply, {:error, WorkerProtocol.error(reason)}, socket} + end end end @@ -561,7 +564,8 @@ defmodule ReencodarrWeb.WorkerChannel do message: "CRF search failed" ) - clear_assigned_video(worker_id, socket) + _ = Media.mark_as_failed(video) + {:ok, clear_assigned_video(worker_id, socket)} end defp ensure_active_video(socket, video_id) do @@ -671,61 +675,85 @@ defmodule ReencodarrWeb.WorkerChannel do {:reply, {:ok, WorkerProtocol.event_ack("crf_search_result")}, socket} crf -> - case Media.mark_vmaf_as_chosen(video.id, crf) do - {:ok, _} -> - _ = Media.mark_as_crf_searched(video) - _ = Media.resolve_crf_search_failures(video.id) - - {:reply, {:ok, WorkerProtocol.event_ack("crf_search_result")}, - clear_assigned_video(worker_id, socket)} - - {:error, _reason} -> - _ = - Media.record_video_failure(video, :crf_search, :validation, - code: "no_chosen_vmaf", - message: "CRF search reported a chosen CRF but no matching VMAF row was found", - context: %{video_id: video.id, chosen_crf: crf} - ) - - _ = Media.mark_as_failed(video) - - {:reply, {:error, WorkerProtocol.error(:invalid_crf_search_result)}, - clear_assigned_video(worker_id, socket)} - end + mark_reported_chosen_crf(worker_id, socket, video, crf) + end + end + + defp mark_reported_chosen_crf(worker_id, socket, video, crf) do + case Media.mark_vmaf_as_chosen(video.id, crf) do + {:ok, _} -> + reply_after_successful_crf_search_result(worker_id, socket, video) + + {:error, _reason} -> + _ = + Media.record_video_failure(video, :crf_search, :validation, + code: "no_chosen_vmaf", + message: "CRF search reported a chosen CRF but no matching VMAF row was found", + context: %{video_id: video.id, chosen_crf: crf} + ) + + _ = Media.mark_as_failed(video) + + {:reply, {:error, WorkerProtocol.error(:invalid_crf_search_result)}, + clear_assigned_video(worker_id, socket)} + end + end + + defp reply_after_successful_crf_search_result(worker_id, socket, video) do + case finalize_successful_crf_search(worker_id, socket, video) do + {:ok, socket} -> + {:reply, {:ok, WorkerProtocol.event_ack("crf_search_result")}, socket} + + {:error, reason, socket} -> + {:reply, {:error, WorkerProtocol.error(reason)}, socket} end end defp finish_successful_crf_search(worker_id, socket, video, chosen_crf) do cond do is_number(chosen_crf) -> - _ = Media.mark_vmaf_as_chosen(video.id, chosen_crf) - finalize_successful_crf_search(worker_id, socket, video) + case Media.mark_vmaf_as_chosen(video.id, chosen_crf) do + {:ok, _vmaf_id} -> + finalize_successful_crf_search(worker_id, socket, video) + + {:error, reason} -> + fail_invalid_successful_crf_search(worker_id, socket, video, reason) + end Media.chosen_vmaf_exists?(video) -> finalize_successful_crf_search(worker_id, socket, video) true -> - _ = - Media.record_video_failure(video, :crf_search, :validation, - code: "no_chosen_vmaf", - message: "CRF search completed successfully but no VMAF was marked as chosen", - context: %{video_id: video.id} - ) - - _ = Media.mark_as_failed(video) - clear_assigned_video(worker_id, socket) + fail_invalid_successful_crf_search(worker_id, socket, video, :no_chosen_vmaf) end end defp finalize_successful_crf_search(worker_id, socket, video) do - _ = Media.mark_as_crf_searched(video) - _ = Media.resolve_crf_search_failures(video.id) - clear_assigned_video(worker_id, socket) + case Media.mark_as_crf_searched(video) do + {:ok, _video} -> + _ = Media.resolve_crf_search_failures(video.id) + {:ok, clear_assigned_video(worker_id, socket)} + + {:error, reason} -> + fail_invalid_successful_crf_search(worker_id, socket, video, reason) + end + end + + defp fail_invalid_successful_crf_search(worker_id, socket, video, reason) do + _ = + Media.record_video_failure(video, :crf_search, :validation, + code: "invalid_successful_crf_search", + message: "CRF search completed successfully but could not choose a VMAF", + context: %{video_id: video.id, reason: inspect(reason)} + ) + + _ = Media.mark_as_failed(video) + {:error, :invalid_crf_search_result, clear_assigned_video(worker_id, socket)} end defp finish_cancelled_crf_search(worker_id, socket, video, _reason) do _ = Media.mark_as_analyzed(video) - clear_assigned_video(worker_id, socket) + {:ok, clear_assigned_video(worker_id, socket)} end defp clear_assigned_video(worker_id, socket) do diff --git a/test/reencodarr/ab_av1/worker_job_state_machine_test.exs b/test/reencodarr/ab_av1/worker_job_state_machine_test.exs new file mode 100644 index 00000000..b1beed21 --- /dev/null +++ b/test/reencodarr/ab_av1/worker_job_state_machine_test.exs @@ -0,0 +1,80 @@ +defmodule Reencodarr.AbAv1.WorkerJobStateMachineTest do + use ExUnit.Case, async: true + + alias Reencodarr.AbAv1.WorkerJobStateMachine + + test "moves a worker through input transfer into CRF search without dropping the active video" do + session = session() + + assert {:ok, session} = WorkerJobStateMachine.assign_video(session, 123, :receiving_input) + assert session.phase == :receiving_input + assert session.active_video_id == 123 + + assert {:ok, session} = + WorkerJobStateMachine.set_transfer_progress(session, %{ + video_id: 123, + percent: 100.0 + }) + + assert session.phase == :receiving_input + assert session.transfer_progress.percent == 100.0 + + assert {:ok, session} = WorkerJobStateMachine.finish_transfer(session) + assert session.phase == :crf_searching + assert session.active_video_id == 123 + assert is_nil(session.transfer_progress) + end + + test "CRF progress wins over stale transfer progress" do + session = + session() + |> Map.merge(%{ + phase: :receiving_input, + active_video_id: 123, + transfer_progress: %{video_id: 123, percent: 100.0} + }) + + assert {:ok, session} = + WorkerJobStateMachine.set_crf_search_progress(session, %{ + video_id: 123, + percent: 75.0 + }) + + assert session.phase == :crf_searching + assert session.active_video_id == 123 + assert is_nil(session.transfer_progress) + assert session.crf_search_progress.percent == 75.0 + end + + test "clearing active work is the only transition that makes the worker idle" do + session = + session() + |> Map.merge(%{ + phase: :crf_searching, + active_video_id: 123, + crf_search_progress: %{video_id: 123, percent: 75.0} + }) + + assert {:ok, session} = WorkerJobStateMachine.clear_video(session) + assert session.phase == :idle + assert is_nil(session.active_video_id) + assert is_nil(session.crf_search_progress) + end + + defp session do + %{ + server_worker_id: "worker-server-1", + client_worker_id: "worker-client-1", + version: "0.11.4", + protocol_version: 1, + capabilities: %{"crf_search" => true}, + phase: :idle, + active_video_id: nil, + transfer_progress: nil, + crf_search_progress: nil, + resource_usage: nil, + connected_at: DateTime.utc_now(), + last_seen_at: DateTime.utc_now() + } + end +end diff --git a/test/reencodarr_web/channels/worker_channel_test.exs b/test/reencodarr_web/channels/worker_channel_test.exs index 484a93c5..810916ec 100644 --- a/test/reencodarr_web/channels/worker_channel_test.exs +++ b/test/reencodarr_web/channels/worker_channel_test.exs @@ -810,6 +810,10 @@ defmodule ReencodarrWeb.WorkerChannelTest do assert Media.get_video(video_id).chosen_vmaf_id != nil assert WorkerSessions.get(socket.assigns.worker_id).active_video_id == nil + assert_reply push(socket, "pull_work", %{}), + :ok, + %{status: "no_work"} + assert_reply push(socket, "crf_search_result", %{ "video_id" => video_id, "results" => [ @@ -942,11 +946,15 @@ defmodule ReencodarrWeb.WorkerChannelTest do "video_id" => video_id, "result" => "ok" }), - :ok, - %{accepted: true, event: "crf_search_completed"} + :error, + %{reason: "invalid_crf_search_result"} assert Media.get_video(video_id).state == :failed assert is_nil(Media.get_video(video_id).chosen_vmaf_id) + + assert_reply push(socket, "pull_work", %{}), + :ok, + %{status: "no_work"} after Application.delete_env(:reencodarr, :worker_token) end From 8398eaf74ba09e2a084d923f2bd2a93f1027fca9 Mon Sep 17 00:00:00 2001 From: Mika Cohen Date: Wed, 8 Jul 2026 20:35:10 -0600 Subject: [PATCH 57/70] stop resending active worker crf jobs --- lib/reencodarr/ab_av1/worker_sessions.ex | 19 +++++++- lib/reencodarr_web/channels/worker_channel.ex | 14 ++++-- .../ab_av1/worker_sessions_test.exs | 16 ++++++- .../channels/worker_channel_test.exs | 47 +++++++++++++++++-- 4 files changed, 86 insertions(+), 10 deletions(-) diff --git a/lib/reencodarr/ab_av1/worker_sessions.ex b/lib/reencodarr/ab_av1/worker_sessions.ex index 98e65eca..5038d532 100644 --- a/lib/reencodarr/ab_av1/worker_sessions.ex +++ b/lib/reencodarr/ab_av1/worker_sessions.ex @@ -425,6 +425,23 @@ defmodule Reencodarr.AbAv1.WorkerSessions do end end + defp preserve_dispatched_active_video(%{active_video_id: nil}), do: :ok + + defp preserve_dispatched_active_video(%{active_video_id: video_id}) do + case Media.get_video(video_id) do + %Media.Video{state: :crf_searching, crf_search_worker_id: dispatch_id} + when is_binary(dispatch_id) -> + :ok + + %Media.Video{state: :crf_searching} = video -> + _ = VideoStateMachine.mark_as_analyzed(video) + :ok + + _ -> + :ok + end + end + defp resumable_active_video_id(nil), do: nil defp resumable_active_video_id(video_id) do @@ -466,7 +483,7 @@ defmodule Reencodarr.AbAv1.WorkerSessions do end) Enum.each(expired_sessions, fn session -> - requeue_active_video(session) + preserve_dispatched_active_video(session) :ok = drop_session(session.server_worker_id) end) diff --git a/lib/reencodarr_web/channels/worker_channel.ex b/lib/reencodarr_web/channels/worker_channel.ex index e43a71dc..f6c90825 100644 --- a/lib/reencodarr_web/channels/worker_channel.ex +++ b/lib/reencodarr_web/channels/worker_channel.ex @@ -200,7 +200,7 @@ defmodule ReencodarrWeb.WorkerChannel do end defp maybe_resume_mode( - %{transfer_progress: transfer_progress, crf_search_progress: _crf_search_progress}, + %{phase: :receiving_input, transfer_progress: transfer_progress}, request_mode ) do if should_resend_transfer?(transfer_progress), @@ -208,6 +208,10 @@ defmodule ReencodarrWeb.WorkerChannel do else: request_mode end + defp maybe_resume_mode(%{phase: :crf_searching}, :resend_input), do: :resend_input + defp maybe_resume_mode(%{phase: :crf_searching}, _request_mode), do: :resume_only + defp maybe_resume_mode(_session, request_mode), do: request_mode + defp should_resend_transfer?(nil), do: true defp should_resend_transfer?(%{bytes_sent: bytes_sent, total_bytes: total_bytes}) @@ -253,11 +257,11 @@ defmodule ReencodarrWeb.WorkerChannel do end end - defp resume_dispatched_work(worker_id, socket, _request_mode) do + defp resume_dispatched_work(worker_id, socket, request_mode) do case Media.get_worker_crf_searching_video(worker_dispatch_id(socket)) do %Media.Video{} = video -> with {:ok, socket} <- ensure_resumable_active_video(socket, video.id) do - reply_for_active_work(worker_id, socket, video, :resend_input) + reply_for_active_work(worker_id, socket, video, request_mode) end nil -> @@ -442,6 +446,10 @@ defmodule ReencodarrWeb.WorkerChannel do video_id -> case Media.get_video(video_id) do + %Media.Video{state: :crf_searching, crf_search_worker_id: dispatch_id} + when is_binary(dispatch_id) -> + :ok + %Media.Video{state: :crf_searching} = video -> _ = Media.mark_as_analyzed(video) :ok diff --git a/test/reencodarr/ab_av1/worker_sessions_test.exs b/test/reencodarr/ab_av1/worker_sessions_test.exs index 544db0c0..af1a3203 100644 --- a/test/reencodarr/ab_av1/worker_sessions_test.exs +++ b/test/reencodarr/ab_av1/worker_sessions_test.exs @@ -38,6 +38,10 @@ defmodule Reencodarr.AbAv1.WorkerSessionsTest do test "expires stale active sessions by requeueing only in-progress videos" do {:ok, active_video} = Fixtures.video_fixture(%{state: :crf_searching}) + + {:ok, dispatched_video} = + Fixtures.video_fixture(%{state: :crf_searching, crf_search_worker_id: "worker-client-3"}) + {:ok, completed_video} = Fixtures.video_fixture(%{state: :crf_searched}) _vmaf = Fixtures.vmaf_fixture(%{video_id: completed_video.id, crf: 28.0, score: 96.4}) assert {:ok, _} = Media.mark_vmaf_as_chosen(completed_video.id, 28.0) @@ -53,11 +57,21 @@ defmodule Reencodarr.AbAv1.WorkerSessionsTest do ) ) + assert {:ok, _session} = + WorkerSessions.register( + worker_session_attrs( + server_worker_id: "worker-server-3", + client_worker_id: "worker-client-3" + ) + ) + assert {:ok, _session} = WorkerSessions.assign_video("worker-server-1", active_video.id) assert {:ok, _session} = WorkerSessions.assign_video("worker-server-2", completed_video.id) + assert {:ok, _session} = WorkerSessions.assign_video("worker-server-3", dispatched_video.id) - assert {:ok, [_session_one, _session_two]} = WorkerSessions.expire_stale(0) + assert {:ok, [_session_one, _session_two, _session_three]} = WorkerSessions.expire_stale(0) assert Media.get_video(active_video.id).state == :analyzed + assert Media.get_video(dispatched_video.id).state == :crf_searching assert Media.get_video(completed_video.id).state == :crf_searched assert WorkerSessions.list() == [] end diff --git a/test/reencodarr_web/channels/worker_channel_test.exs b/test/reencodarr_web/channels/worker_channel_test.exs index 810916ec..d9ecda9f 100644 --- a/test/reencodarr_web/channels/worker_channel_test.exs +++ b/test/reencodarr_web/channels/worker_channel_test.exs @@ -447,7 +447,7 @@ defmodule ReencodarrWeb.WorkerChannelTest do Application.delete_env(:reencodarr, :worker_token) end - test "returns in-progress work when reconnecting worker asks again before any progress" do + test "returns in-progress work when reconnecting dispatched worker asks again" do token = "test-worker-token" Application.put_env(:reencodarr, :worker_token, token) @@ -470,6 +470,43 @@ defmodule ReencodarrWeb.WorkerChannelTest do %{accepted: true, protocol_version: 1} assert_reply push(socket, "pull_work", %{}), + :ok, + %{ + status: "job_in_progress", + video_id: ^video_id, + crf_search_args: crf_search_args + } + + assert "crf-search" in crf_search_args + refute_push "transfer_started", _, 50 + end) + after + Application.delete_env(:reencodarr, :worker_token) + end + + test "resends dispatched work when reconnecting worker reports missing input" do + token = "test-worker-token" + Application.put_env(:reencodarr, :worker_token, token) + + with_temp_file("abcdefgh", ".mkv", fn path -> + {:ok, video} = + Fixtures.video_fixture(%{ + path: path, + size: 8, + state: :crf_searching, + crf_search_worker_id: "worker-a" + }) + + video_id = video.id + + assert {:ok, socket} = connect(WorkerSocket, %{"token" => token}) + assert {:ok, _join_payload, socket} = subscribe_and_join(socket, "workers:crf_search") + + assert_reply push(socket, "announce", announce_payload(worker_id: "worker-a")), + :ok, + %{accepted: true, protocol_version: 1} + + assert_reply push(socket, "pull_work", %{"input_missing" => true}), :ok, %{ status: "job_assigned", @@ -517,7 +554,7 @@ defmodule ReencodarrWeb.WorkerChannelTest do assert_reply push(socket, "pull_work", %{}), :ok, %{ - status: "job_assigned", + status: "job_in_progress", video_id: ^video_id, crf_search_args: in_progress_args } @@ -595,7 +632,7 @@ defmodule ReencodarrWeb.WorkerChannelTest do Application.delete_env(:reencodarr, :worker_token) end - test "resends input when the dispatched work is found but session state is gone" do + test "returns in-progress when the dispatched work is found but session state is gone" do token = "test-worker-token" Application.put_env(:reencodarr, :worker_token, token) @@ -624,13 +661,13 @@ defmodule ReencodarrWeb.WorkerChannelTest do assert_reply push(socket, "pull_work", %{}), :ok, %{ - status: "job_assigned", + status: "job_in_progress", video_id: ^video_id, crf_search_args: crf_search_args } assert "crf-search" in crf_search_args - assert_push "transfer_started", %{status: "transfer_started", video_id: ^video_id} + refute_push "transfer_started", _, 50 end) after Application.delete_env(:reencodarr, :worker_token) From c1ea3e1d5530ce8c54d183ead33e137e1e8e14f4 Mon Sep 17 00:00:00 2001 From: Mika Cohen Date: Wed, 8 Jul 2026 20:51:42 -0600 Subject: [PATCH 58/70] fail orphaned worker crf dispatches --- lib/reencodarr_web/channels/worker_channel.ex | 33 +++++++++++-------- .../channels/worker_channel_test.exs | 30 +++++++---------- 2 files changed, 31 insertions(+), 32 deletions(-) diff --git a/lib/reencodarr_web/channels/worker_channel.ex b/lib/reencodarr_web/channels/worker_channel.ex index f6c90825..9e082c8c 100644 --- a/lib/reencodarr_web/channels/worker_channel.ex +++ b/lib/reencodarr_web/channels/worker_channel.ex @@ -234,16 +234,7 @@ defmodule ReencodarrWeb.WorkerChannel do attach_active_video(socket, video_id) end - defp attach_announced_work(socket, client_worker_id, _session) do - case Media.get_worker_crf_searching_video(client_worker_id) do - %Media.Video{} = video -> - _ = WorkerSessions.assign_video(socket.assigns.worker_id, video.id, :crf_searching) - attach_active_video(socket, video.id) - - nil -> - socket - end - end + defp attach_announced_work(socket, _client_worker_id, _session), do: socket defp attach_active_video(socket, video_id) do case Media.get_video(video_id) do @@ -260,15 +251,31 @@ defmodule ReencodarrWeb.WorkerChannel do defp resume_dispatched_work(worker_id, socket, request_mode) do case Media.get_worker_crf_searching_video(worker_dispatch_id(socket)) do %Media.Video{} = video -> - with {:ok, socket} <- ensure_resumable_active_video(socket, video.id) do - reply_for_active_work(worker_id, socket, video, request_mode) - end + resume_dispatched_work(worker_id, socket, video, request_mode) nil -> :none end end + defp resume_dispatched_work(worker_id, socket, video, :resend_input) do + with {:ok, socket} <- ensure_resumable_active_video(socket, video.id) do + reply_for_active_work(worker_id, socket, video, :resend_input) + end + end + + defp resume_dispatched_work(worker_id, socket, video, :resume_only) do + _ = + Media.record_video_failure(video, :crf_search, :crf_optimization, + code: "worker_disconnected", + message: "Worker disconnected before reporting CRF search completion", + context: %{video_id: video.id, worker_id: worker_dispatch_id(socket)} + ) + + socket = clear_assigned_video(worker_id, socket) + claim_work(worker_id, socket) + end + defp handle_transfer_progress(payload, %{assigns: %{worker_id: worker_id}} = socket) do with {:ok, progress} <- WorkerProtocol.parse_transfer_progress(payload), :ok <- ensure_active_video(socket, progress.video_id) do diff --git a/test/reencodarr_web/channels/worker_channel_test.exs b/test/reencodarr_web/channels/worker_channel_test.exs index d9ecda9f..7169c713 100644 --- a/test/reencodarr_web/channels/worker_channel_test.exs +++ b/test/reencodarr_web/channels/worker_channel_test.exs @@ -360,7 +360,7 @@ defmodule ReencodarrWeb.WorkerChannelTest do %{accepted: true, protocol_version: 1} session = WorkerSessions.get(server_worker_id) - assert session.active_video_id == video_id + assert is_nil(session.active_video_id) assert_reply push(socket, "crf_search_progress", %{ "video_id" => video_id, @@ -447,7 +447,7 @@ defmodule ReencodarrWeb.WorkerChannelTest do Application.delete_env(:reencodarr, :worker_token) end - test "returns in-progress work when reconnecting dispatched worker asks again" do + test "fails orphaned dispatched work when reconnecting worker asks for new work" do token = "test-worker-token" Application.put_env(:reencodarr, :worker_token, token) @@ -469,15 +469,13 @@ defmodule ReencodarrWeb.WorkerChannelTest do :ok, %{accepted: true, protocol_version: 1} - assert_reply push(socket, "pull_work", %{}), - :ok, - %{ - status: "job_in_progress", - video_id: ^video_id, - crf_search_args: crf_search_args - } + assert_reply push(socket, "pull_work", %{}), :ok, %{status: "no_work"} - assert "crf-search" in crf_search_args + assert Media.get_video(video_id).state == :failed + assert [failure] = Media.get_video_failures(video_id) + assert failure.failure_stage == :crf_search + assert failure.failure_category == :crf_optimization + assert failure.failure_code == "worker_disconnected" refute_push "transfer_started", _, 50 end) after @@ -632,7 +630,7 @@ defmodule ReencodarrWeb.WorkerChannelTest do Application.delete_env(:reencodarr, :worker_token) end - test "returns in-progress when the dispatched work is found but session state is gone" do + test "does not restart dispatched work when the session state is gone" do token = "test-worker-token" Application.put_env(:reencodarr, :worker_token, token) @@ -658,15 +656,9 @@ defmodule ReencodarrWeb.WorkerChannelTest do :ok, %{accepted: true, protocol_version: 1} - assert_reply push(socket, "pull_work", %{}), - :ok, - %{ - status: "job_in_progress", - video_id: ^video_id, - crf_search_args: crf_search_args - } + assert_reply push(socket, "pull_work", %{}), :ok, %{status: "no_work"} - assert "crf-search" in crf_search_args + assert Media.get_video(video_id).state == :failed refute_push "transfer_started", _, 50 end) after From 85397c39ec52d3774546d076837d95872a3ba58c Mon Sep 17 00:00:00 2001 From: Mika Cohen Date: Wed, 8 Jul 2026 21:06:41 -0600 Subject: [PATCH 59/70] keep worker transfers in receiving state --- lib/reencodarr_web/channels/worker_channel.ex | 15 +++++++++------ .../channels/worker_channel_test.exs | 4 ++-- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/lib/reencodarr_web/channels/worker_channel.ex b/lib/reencodarr_web/channels/worker_channel.ex index 9e082c8c..a967eb44 100644 --- a/lib/reencodarr_web/channels/worker_channel.ex +++ b/lib/reencodarr_web/channels/worker_channel.ex @@ -259,7 +259,7 @@ defmodule ReencodarrWeb.WorkerChannel do end defp resume_dispatched_work(worker_id, socket, video, :resend_input) do - with {:ok, socket} <- ensure_resumable_active_video(socket, video.id) do + with {:ok, socket} <- ensure_resumable_active_video(socket, video.id, :receiving_input) do reply_for_active_work(worker_id, socket, video, :resend_input) end end @@ -591,13 +591,13 @@ defmodule ReencodarrWeb.WorkerChannel do end end - defp ensure_resumable_active_video(socket, video_id) do + defp ensure_resumable_active_video(socket, video_id, phase \\ :crf_searching) do case socket.assigns[:current_video_id] do ^video_id -> {:ok, socket} nil -> - resume_active_video(socket, video_id) + resume_active_video(socket, video_id, phase) _other -> {:error, :unknown_worker_session} @@ -628,12 +628,16 @@ defmodule ReencodarrWeb.WorkerChannel do end end - defp resume_active_video(%{assigns: %{worker_id: worker_id}} = socket, video_id) do + defp resume_active_video( + %{assigns: %{worker_id: worker_id}} = socket, + video_id, + phase + ) do with %Media.Video{state: :crf_searching, crf_search_worker_id: dispatch_id} = video when is_binary(dispatch_id) <- Media.get_video(video_id), true <- dispatch_id == worker_dispatch_id(socket), false <- assigned_to_other_worker?(worker_id, video_id), - {:ok, _session} <- WorkerSessions.assign_video(worker_id, video_id, :crf_searching) do + {:ok, _session} <- WorkerSessions.assign_video(worker_id, video_id, phase) do {:ok, socket |> assign(:current_video_id, video_id) @@ -889,7 +893,6 @@ defmodule ReencodarrWeb.WorkerChannel do :eof -> close_transfer_stream(io_device) - _ = WorkerSessions.finish_transfer(socket.assigns.worker_id) push( socket, diff --git a/test/reencodarr_web/channels/worker_channel_test.exs b/test/reencodarr_web/channels/worker_channel_test.exs index 7169c713..6a171460 100644 --- a/test/reencodarr_web/channels/worker_channel_test.exs +++ b/test/reencodarr_web/channels/worker_channel_test.exs @@ -770,8 +770,8 @@ defmodule ReencodarrWeb.WorkerChannelTest do session = WorkerSessions.get(socket.assigns.worker_id) assert session.active_video_id == assigned_video_id - assert session.phase == :crf_searching - assert is_nil(session.transfer_progress) + assert session.phase == :receiving_input + assert session.transfer_progress.video_id == assigned_video_id end) after Application.delete_env(:reencodarr, :worker_token) From 7e40a543f39777939982ce408d374d2fb858bfee Mon Sep 17 00:00:00 2001 From: Mika Cohen Date: Wed, 8 Jul 2026 21:20:40 -0600 Subject: [PATCH 60/70] avoid loading all codecs for dashboard chart --- lib/reencodarr/media/chart_queries.ex | 37 +++++++++++++++------------ 1 file changed, 20 insertions(+), 17 deletions(-) diff --git a/lib/reencodarr/media/chart_queries.ex b/lib/reencodarr/media/chart_queries.ex index 961a2583..8fcd7c5e 100644 --- a/lib/reencodarr/media/chart_queries.ex +++ b/lib/reencodarr/media/chart_queries.ex @@ -1,8 +1,6 @@ defmodule Reencodarr.Media.ChartQueries do @moduledoc "Database queries for dashboard chart data." - import Ecto.Query - alias Reencodarr.Media.Video alias Reencodarr.Repo @vmaf_bins [ @@ -133,22 +131,27 @@ defmodule Reencodarr.Media.ChartQueries do @doc "Get primary codec distribution across all videos (top 8)." def codec_distribution do - from(v in Video, - where: not is_nil(v.video_codecs) and v.video_codecs != [], - select: v.video_codecs - ) - |> Repo.all() - |> Enum.reduce(%{}, fn codecs, acc -> - codec = - case codecs do - [first | _] -> normalize_codec(first) - _ -> "unknown" - end + case Repo.query( + """ + SELECT json_extract(video_codecs, '$[0]') AS codec, COUNT(*) + FROM videos + WHERE video_codecs IS NOT NULL + AND json_array_length(video_codecs) > 0 + GROUP BY codec + """, + [] + ) do + {:ok, %{rows: rows}} -> + rows + |> Enum.reduce(%{}, fn [codec, count], acc -> + Map.update(acc, normalize_codec(codec), count, &(&1 + count)) + end) + |> Enum.sort_by(fn {_, count} -> -count end) + |> Enum.take(8) - Map.update(acc, codec, 1, &(&1 + 1)) - end) - |> Enum.sort_by(fn {_, count} -> -count end) - |> Enum.take(8) + {:error, error} -> + raise error + end end defp normalize_codec(codec) when is_binary(codec) do From daf2c8cb42f9f2d57066fd4b390acd27fd0304ec Mon Sep 17 00:00:00 2001 From: Mika Cohen Date: Wed, 8 Jul 2026 21:30:18 -0600 Subject: [PATCH 61/70] force crf queue index for dashboard preview --- lib/reencodarr/media/video_queries.ex | 65 +++++++++++++++------------ 1 file changed, 36 insertions(+), 29 deletions(-) diff --git a/lib/reencodarr/media/video_queries.ex b/lib/reencodarr/media/video_queries.ex index e907c71b..d5f17e8e 100644 --- a/lib/reencodarr/media/video_queries.ex +++ b/lib/reencodarr/media/video_queries.ex @@ -18,16 +18,12 @@ defmodule Reencodarr.Media.VideoQueries do """ @spec videos_for_crf_search(integer(), keyword()) :: [Video.t()] def videos_for_crf_search(limit \\ 10, opts \\ []) do - # Simplified query - just check state, let Rules filter codecs at encode time - Repo.all( - from(v in Video, - where: v.state == :analyzed, - order_by: [desc: v.priority, desc: v.bitrate, desc: v.size, asc: v.updated_at], - limit: ^limit, - select: v - ), - opts - ) + ids = crf_search_queue_candidate_ids(limit, opts) + + Video + |> where([v], v.id in ^ids) + |> Repo.all(opts) + |> sort_by_ids(ids) end @doc """ @@ -189,16 +185,7 @@ defmodule Reencodarr.Media.VideoQueries do end defp claim_next_video_for_crf_search_in_tx(opts) do - candidate_ids = - from(v in Video, - where: v.state == :analyzed, - order_by: [desc: v.priority, desc: v.bitrate, desc: v.size, asc: v.updated_at], - limit: 10, - select: v.id - ) - |> Repo.all(opts) - - claim_next_video_for_crf_search_in_tx(candidate_ids, opts) + claim_next_video_for_crf_search_in_tx(crf_search_queue_candidate_ids(10, opts), opts) end defp claim_next_video_for_crf_search_in_tx([], _opts), do: :none @@ -241,15 +228,7 @@ defmodule Reencodarr.Media.VideoQueries do """ @spec videos_for_crf_search_preview(integer(), keyword()) :: [map()] def videos_for_crf_search_preview(limit \\ 10, opts \\ []) do - Repo.all( - from(v in Video, - where: v.state == :analyzed, - order_by: [desc: v.priority, desc: v.bitrate, desc: v.size, asc: v.updated_at], - limit: ^limit, - select: %{id: v.id, path: v.path} - ), - opts - ) + crf_search_queue_preview_rows(limit, opts) end @doc """ @@ -305,4 +284,32 @@ defmodule Reencodarr.Media.VideoQueries do opts ) end + + defp crf_search_queue_candidate_ids(limit, opts) do + crf_search_queue_rows("id", limit, opts) + |> Enum.map(fn [id] -> id end) + end + + defp crf_search_queue_preview_rows(limit, opts) do + crf_search_queue_rows("id, path", limit, opts) + |> Enum.map(fn [id, path] -> %{id: id, path: path} end) + end + + defp crf_search_queue_rows(select, limit, opts) do + sql = """ + SELECT #{select} + FROM videos INDEXED BY videos_crf_search_queue_index + WHERE state = 'analyzed' + ORDER BY priority DESC, bitrate DESC, size DESC, updated_at ASC + LIMIT ? + """ + + %{rows: rows} = Repo.query!(sql, [limit], opts) + rows + end + + defp sort_by_ids(videos, ids) do + videos_by_id = Map.new(videos, &{&1.id, &1}) + Enum.map(ids, &Map.fetch!(videos_by_id, &1)) + end end From 47701d92f930da1741009ce51b7ab09fb3197e34 Mon Sep 17 00:00:00 2001 From: Mika Cohen Date: Wed, 8 Jul 2026 21:52:13 -0600 Subject: [PATCH 62/70] keep worker sessions alive during transfer --- lib/reencodarr_web/channels/worker_channel.ex | 31 +++++++++++++++++++ .../channels/worker_channel_test.exs | 10 ++++++ 2 files changed, 41 insertions(+) diff --git a/lib/reencodarr_web/channels/worker_channel.ex b/lib/reencodarr_web/channels/worker_channel.ex index a967eb44..30e83fe1 100644 --- a/lib/reencodarr_web/channels/worker_channel.ex +++ b/lib/reencodarr_web/channels/worker_channel.ex @@ -858,6 +858,31 @@ defmodule ReencodarrWeb.WorkerChannel do } end + defp outbound_transfer_progress(socket, video, bytes_sent, chunk_index) do + total_bytes = socket.assigns.transfer_total_bytes + + %WorkerProtocol.TransferProgress{ + job_id: socket.assigns.transfer_id, + video_id: video.id, + transfer_id: socket.assigns.transfer_id, + filename: Path.basename(video.path), + percent: transfer_percent(bytes_sent, total_bytes), + bytes_sent: bytes_sent, + total_bytes: total_bytes, + bytes_per_second: nil, + eta: nil, + chunk_index: chunk_index, + total_chunks: socket.assigns.transfer_total_chunks + } + end + + defp transfer_percent(bytes_sent, total_bytes) + when is_integer(bytes_sent) and is_integer(total_bytes) and total_bytes > 0 do + bytes_sent / total_bytes * 100.0 + end + + defp transfer_percent(_bytes_sent, _total_bytes), do: 0.0 + defp read_transfer_chunk(%{assigns: %{current_video_id: video_id}} = socket, io_device) do video = Media.get_video(video_id) @@ -870,6 +895,12 @@ defmodule ReencodarrWeb.WorkerChannel do chunk_index = socket.assigns.transfer_chunk_index bytes_sent = socket.assigns.transfer_bytes_sent + byte_size(chunk) + _ = + WorkerSessions.set_transfer_progress( + socket.assigns.worker_id, + outbound_transfer_progress(socket, video, bytes_sent, chunk_index + 1) + ) + push( socket, "transfer_chunk", diff --git a/test/reencodarr_web/channels/worker_channel_test.exs b/test/reencodarr_web/channels/worker_channel_test.exs index 6a171460..de1f0871 100644 --- a/test/reencodarr_web/channels/worker_channel_test.exs +++ b/test/reencodarr_web/channels/worker_channel_test.exs @@ -760,6 +760,16 @@ defmodule ReencodarrWeb.WorkerChannelTest do data: ^second_chunk }} = WorkerProtocol.parse_transfer_chunk_frame(second_frame) + session = WorkerSessions.get(socket.assigns.worker_id) + assert session.active_video_id == assigned_video_id + assert session.phase == :receiving_input + assert session.transfer_progress.video_id == assigned_video_id + assert session.transfer_progress.bytes_sent == content_size + assert session.transfer_progress.total_bytes == content_size + assert session.transfer_progress.chunk_index == 2 + assert session.transfer_progress.total_chunks == 2 + assert session.transfer_progress.percent == 100.0 + assert_push "transfer_complete", %{ status: "transfer_complete", video_id: ^assigned_video_id, From e89bd33d6c2a4d6be8771522cd1cc519e31bd702 Mon Sep 17 00:00:00 2001 From: Mika Cohen Date: Wed, 8 Jul 2026 22:14:28 -0600 Subject: [PATCH 63/70] make worker job phase authoritative --- .../ab_av1/worker_job_state_machine.ex | 67 +++++++++++++------ lib/reencodarr/ab_av1/worker_sessions.ex | 23 +++++-- lib/reencodarr/diagnostics.ex | 8 +-- lib/reencodarr_web/channels/worker_channel.ex | 22 +++--- lib/reencodarr_web/live/workers_live.ex | 35 +++++----- .../ab_av1/worker_job_state_machine_test.exs | 42 +++++++++++- .../ab_av1/worker_sessions_test.exs | 28 +++++++- .../channels/worker_channel_test.exs | 29 ++++---- .../reencodarr_web/live/workers_live_test.exs | 22 ++---- 9 files changed, 187 insertions(+), 89 deletions(-) diff --git a/lib/reencodarr/ab_av1/worker_job_state_machine.ex b/lib/reencodarr/ab_av1/worker_job_state_machine.ex index 098a8240..6dde7075 100644 --- a/lib/reencodarr/ab_av1/worker_job_state_machine.ex +++ b/lib/reencodarr/ab_av1/worker_job_state_machine.ex @@ -5,13 +5,14 @@ defmodule Reencodarr.AbAv1.WorkerJobStateMachine do require Logger - @type phase :: :idle | :receiving_input | :crf_searching + @type phase :: :idle | :receiving_input | :input_ready | :crf_searching - @valid_phases [:idle, :receiving_input, :crf_searching] + @valid_phases [:idle, :receiving_input, :input_ready, :crf_searching] @valid_transitions %{ idle: [:receiving_input, :crf_searching], - receiving_input: [:receiving_input, :crf_searching, :idle], + receiving_input: [:receiving_input, :input_ready, :crf_searching, :idle], + input_ready: [:receiving_input, :crf_searching, :idle], crf_searching: [:crf_searching, :receiving_input, :idle] } @@ -44,11 +45,13 @@ defmodule Reencodarr.AbAv1.WorkerJobStateMachine do @spec assign_video(map(), integer(), phase()) :: {:ok, map()} | {:error, :invalid_worker_phase} def assign_video(session, video_id, phase) when is_integer(video_id) do - transition(session, phase, %{ - active_video_id: video_id, - transfer_progress: nil, - crf_search_progress: nil - }) + with :ok <- ensure_same_active_video(session, video_id) do + transition(session, phase, %{ + active_video_id: video_id, + transfer_progress: nil, + crf_search_progress: nil + }) + end end @spec clear_video(map()) :: {:ok, map()} @@ -59,27 +62,49 @@ defmodule Reencodarr.AbAv1.WorkerJobStateMachine do end @spec set_transfer_progress(map(), map()) :: {:ok, map()} | {:error, :invalid_worker_phase} - def set_transfer_progress(session, progress) do - transition(session, :receiving_input, %{ - active_video_id: Map.get(progress, :video_id, session.active_video_id), - transfer_progress: progress, - crf_search_progress: nil - }) + def set_transfer_progress(%{phase: phase} = session, progress) + when phase in [:idle, :receiving_input, :input_ready] do + with {:ok, video_id} <- progress_video_id(session, progress) do + transition(session, :receiving_input, %{ + active_video_id: video_id, + transfer_progress: progress, + crf_search_progress: nil + }) + end end - @spec finish_transfer(map()) :: {:ok, map()} + def set_transfer_progress(_session, _progress), do: {:error, :invalid_worker_phase} + + @spec finish_transfer(map()) :: {:ok, map()} | {:error, :invalid_worker_phase} def finish_transfer(%{active_video_id: nil} = session), do: clear_video(session) def finish_transfer(session) do - {:ok, %{session | phase: :crf_searching, transfer_progress: nil}} + transition(session, :input_ready) end @spec set_crf_search_progress(map(), map()) :: {:ok, map()} | {:error, :invalid_worker_phase} def set_crf_search_progress(session, progress) do - transition(session, :crf_searching, %{ - active_video_id: Map.get(progress, :video_id, session.active_video_id), - transfer_progress: nil, - crf_search_progress: progress - }) + with {:ok, video_id} <- progress_video_id(session, progress) do + transition(session, :crf_searching, %{ + active_video_id: video_id, + transfer_progress: nil, + crf_search_progress: progress + }) + end + end + + defp ensure_same_active_video(%{active_video_id: nil}, _video_id), do: :ok + defp ensure_same_active_video(%{active_video_id: video_id}, video_id), do: :ok + defp ensure_same_active_video(_session, _video_id), do: {:error, :invalid_worker_phase} + + defp progress_video_id(session, progress) do + progress_video_id = Map.get(progress, :video_id) + + case {session.active_video_id, progress_video_id} do + {video_id, video_id} when is_integer(video_id) -> {:ok, video_id} + {nil, video_id} when is_integer(video_id) -> {:ok, video_id} + {video_id, nil} when is_integer(video_id) -> {:ok, video_id} + _ -> {:error, :invalid_worker_phase} + end end end diff --git a/lib/reencodarr/ab_av1/worker_sessions.ex b/lib/reencodarr/ab_av1/worker_sessions.ex index 5038d532..f3a5be03 100644 --- a/lib/reencodarr/ab_av1/worker_sessions.ex +++ b/lib/reencodarr/ab_av1/worker_sessions.ex @@ -19,7 +19,7 @@ defmodule Reencodarr.AbAv1.WorkerSessions do version: String.t(), protocol_version: pos_integer(), capabilities: map(), - phase: :idle | :receiving_input | :crf_searching, + phase: :idle | :receiving_input | :input_ready | :crf_searching, active_video_id: integer() | nil, transfer_progress: map() | nil, crf_search_progress: map() | nil, @@ -45,7 +45,7 @@ defmodule Reencodarr.AbAv1.WorkerSessions do end def assign_video(server_worker_id, video_id, phase \\ :crf_searching) - when is_integer(video_id) and phase in [:receiving_input, :crf_searching] do + when is_integer(video_id) and phase in [:receiving_input, :input_ready, :crf_searching] do GenServer.call(__MODULE__, {:assign_video, server_worker_id, video_id, phase}) end @@ -350,7 +350,13 @@ defmodule Reencodarr.AbAv1.WorkerSessions do active_video_id = resumable_active_video_id(existing_session.active_video_id) phase = - if(active_video_id, do: Map.get(existing_session, :phase, :crf_searching), else: :idle) + if active_video_id do + existing_session + |> Map.get(:phase, :crf_searching) + |> resumable_phase() + else + :idle + end session = build_session( @@ -366,13 +372,15 @@ defmodule Reencodarr.AbAv1.WorkerSessions do |> Map.put(:phase, phase) |> Map.put( :transfer_progress, - if(active_video_id && phase == :receiving_input, + if(active_video_id && phase in [:receiving_input, :input_ready], do: existing_session.transfer_progress ) ) |> Map.put( :crf_search_progress, - if(active_video_id, do: existing_session.crf_search_progress) + if(active_video_id && phase == :crf_searching, + do: existing_session.crf_search_progress + ) ) |> Map.put(:resource_usage, existing_session.resource_usage) @@ -387,6 +395,11 @@ defmodule Reencodarr.AbAv1.WorkerSessions do defp required_attr(attrs, key), do: Map.fetch(attrs, key) + defp resumable_phase(phase) when phase in [:receiving_input, :input_ready, :crf_searching], + do: phase + + defp resumable_phase(_phase), do: :crf_searching + defp update_session_reply(server_worker_id, state, update_fun) do case lookup_session(server_worker_id) do {:ok, session} -> diff --git a/lib/reencodarr/diagnostics.ex b/lib/reencodarr/diagnostics.ex index c5c06310..3b1fb912 100644 --- a/lib/reencodarr/diagnostics.ex +++ b/lib/reencodarr/diagnostics.ex @@ -670,17 +670,17 @@ defmodule Reencodarr.Diagnostics do defp format_worker_sessions(worker_sessions) when is_list(worker_sessions) do Enum.map_join(worker_sessions, "\n", fn session -> active_video_id = session.active_video_id || "none" - active_video_state = worker_session_state(session.active_video_id) + video_state = worker_session_video_state(session.active_video_id) transfer_progress = format_worker_progress(session.transfer_progress, "transfer") crf_search_progress = format_worker_progress(session.crf_search_progress, "crf") - " #{session.client_worker_id} protocol=#{session.protocol_version} version=#{session.version} video=#{active_video_id} state=#{active_video_state} transfer=#{transfer_progress} crf=#{crf_search_progress} capabilities=#{inspect(session.capabilities)} last_seen=#{DateTime.to_iso8601(session.last_seen_at)}" + " #{session.client_worker_id} protocol=#{session.protocol_version} version=#{session.version} video=#{active_video_id} phase=#{session.phase} video_state=#{video_state} transfer=#{transfer_progress} crf=#{crf_search_progress} capabilities=#{inspect(session.capabilities)} last_seen=#{DateTime.to_iso8601(session.last_seen_at)}" end) end - defp worker_session_state(nil), do: "idle" + defp worker_session_video_state(nil), do: "none" - defp worker_session_state(video_id) do + defp worker_session_video_state(video_id) do case Reencodarr.Media.get_video(video_id) do %Video{state: state} -> Atom.to_string(state) nil -> "missing" diff --git a/lib/reencodarr_web/channels/worker_channel.ex b/lib/reencodarr_web/channels/worker_channel.ex index 30e83fe1..ab75fc77 100644 --- a/lib/reencodarr_web/channels/worker_channel.ex +++ b/lib/reencodarr_web/channels/worker_channel.ex @@ -210,6 +210,8 @@ defmodule ReencodarrWeb.WorkerChannel do defp maybe_resume_mode(%{phase: :crf_searching}, :resend_input), do: :resend_input defp maybe_resume_mode(%{phase: :crf_searching}, _request_mode), do: :resume_only + defp maybe_resume_mode(%{phase: :input_ready}, :resend_input), do: :resend_input + defp maybe_resume_mode(%{phase: :input_ready}, _request_mode), do: :resume_only defp maybe_resume_mode(_session, request_mode), do: request_mode defp should_resend_transfer?(nil), do: true @@ -265,15 +267,9 @@ defmodule ReencodarrWeb.WorkerChannel do end defp resume_dispatched_work(worker_id, socket, video, :resume_only) do - _ = - Media.record_video_failure(video, :crf_search, :crf_optimization, - code: "worker_disconnected", - message: "Worker disconnected before reporting CRF search completion", - context: %{video_id: video.id, worker_id: worker_dispatch_id(socket)} - ) - - socket = clear_assigned_video(worker_id, socket) - claim_work(worker_id, socket) + with {:ok, socket} <- ensure_resumable_active_video(socket, video.id, :crf_searching) do + reply_for_active_work(worker_id, socket, video, :resume_only) + end end defp handle_transfer_progress(payload, %{assigns: %{worker_id: worker_id}} = socket) do @@ -473,6 +469,7 @@ defmodule ReencodarrWeb.WorkerChannel do {:reply, {:error, WorkerProtocol.error(:unknown_worker_session)}, socket} video -> + mark_crf_search_active(socket.assigns.worker_id, video_id) persist_crf_results(video, results) if Enum.any?(results, &Map.get(&1, :chosen, false)) do @@ -512,6 +509,8 @@ defmodule ReencodarrWeb.WorkerChannel do clear_assigned_video(worker_id, socket)} video -> + mark_crf_search_active(worker_id, completion.video_id) + case apply_completion_result( worker_id, socket, @@ -791,6 +790,11 @@ defmodule ReencodarrWeb.WorkerChannel do |> assign(:transfer_started_sent, nil) end + defp mark_crf_search_active(worker_id, video_id) do + _ = WorkerSessions.set_crf_search_progress(worker_id, %{video_id: video_id}) + :ok + end + defp open_transfer_stream( %{assigns: %{current_video_id: video_id, transfer_path: path}} = socket ) do diff --git a/lib/reencodarr_web/live/workers_live.ex b/lib/reencodarr_web/live/workers_live.ex index 9c2a5848..d46adc8b 100644 --- a/lib/reencodarr_web/live/workers_live.ex +++ b/lib/reencodarr_web/live/workers_live.ex @@ -118,7 +118,17 @@ defmodule ReencodarrWeb.WorkersLive do
<%= case worker_phase(worker) do %> <% :receiving_input -> %> - <.transfer_panel worker={worker} video={active_video(worker)} /> + <.transfer_panel + worker={worker} + video={active_video(worker)} + title="Receiving Input" + /> + <% :input_ready -> %> + <.transfer_panel + worker={worker} + video={active_video(worker)} + title="Input Ready" + /> <% :crf_searching -> %> <.crf_search_panel video={worker_crf_video(worker)} @@ -169,7 +179,7 @@ defmodule ReencodarrWeb.WorkersLive do ~H"""
-

Receiving Input

+

{@title}

{format_number(@worker.transfer_progress && @worker.transfer_progress.percent)}% @@ -202,8 +212,8 @@ defmodule ReencodarrWeb.WorkersLive do
{format_transfer_bytes(@worker.transfer_progress)} Chunk {format_chunk_progress(@worker.transfer_progress)} - {format_throughput(@worker.transfer_progress.bytes_per_second)} - ETA {format_eta(@worker.transfer_progress.eta)} + {format_throughput(Map.get(@worker.transfer_progress, :bytes_per_second))} + ETA {format_eta(Map.get(@worker.transfer_progress, :eta))}
<% else %> @@ -217,25 +227,18 @@ defmodule ReencodarrWeb.WorkersLive do case worker_phase(worker) do :idle -> "Idle" :receiving_input -> "Receiving input" - :crf_searching -> crf_search_status(worker) - end - end - - defp crf_search_status(%{active_video_id: video_id}) when is_integer(video_id) do - case Media.get_video(video_id) do - %Media.Video{state: state} -> Atom.to_string(state) - nil -> "missing" + :input_ready -> "Input ready" + :crf_searching -> "CRF search" end end - defp crf_search_status(_worker), do: "CRF search" - defp worker_crf_status(%{phase: :crf_searching}), do: :processing defp worker_crf_status(_worker), do: :idle - defp worker_phase(%{phase: phase}) when phase in [:idle, :receiving_input, :crf_searching], - do: phase + defp worker_phase(%{phase: phase}) + when phase in [:idle, :receiving_input, :input_ready, :crf_searching], + do: phase defp worker_phase(%{active_video_id: nil}), do: :idle diff --git a/test/reencodarr/ab_av1/worker_job_state_machine_test.exs b/test/reencodarr/ab_av1/worker_job_state_machine_test.exs index b1beed21..17a07ff1 100644 --- a/test/reencodarr/ab_av1/worker_job_state_machine_test.exs +++ b/test/reencodarr/ab_av1/worker_job_state_machine_test.exs @@ -20,9 +20,9 @@ defmodule Reencodarr.AbAv1.WorkerJobStateMachineTest do assert session.transfer_progress.percent == 100.0 assert {:ok, session} = WorkerJobStateMachine.finish_transfer(session) - assert session.phase == :crf_searching + assert session.phase == :input_ready assert session.active_video_id == 123 - assert is_nil(session.transfer_progress) + assert session.transfer_progress.percent == 100.0 end test "CRF progress wins over stale transfer progress" do @@ -46,6 +46,44 @@ defmodule Reencodarr.AbAv1.WorkerJobStateMachineTest do assert session.crf_search_progress.percent == 75.0 end + test "progress cannot move an active worker to a different video" do + session = + session() + |> Map.merge(%{ + phase: :receiving_input, + active_video_id: 123, + transfer_progress: %{video_id: 123, percent: 50.0} + }) + + assert {:error, :invalid_worker_phase} = + WorkerJobStateMachine.set_transfer_progress(session, %{ + video_id: 456, + percent: 75.0 + }) + + assert {:error, :invalid_worker_phase} = + WorkerJobStateMachine.set_crf_search_progress(session, %{ + video_id: 456, + percent: 10.0 + }) + end + + test "stale transfer progress cannot move a CRF search back to receiving input" do + session = + session() + |> Map.merge(%{ + phase: :crf_searching, + active_video_id: 123, + crf_search_progress: %{video_id: 123, percent: 25.0} + }) + + assert {:error, :invalid_worker_phase} = + WorkerJobStateMachine.set_transfer_progress(session, %{ + video_id: 123, + percent: 100.0 + }) + end + test "clearing active work is the only transition that makes the worker idle" do session = session() diff --git a/test/reencodarr/ab_av1/worker_sessions_test.exs b/test/reencodarr/ab_av1/worker_sessions_test.exs index af1a3203..24176f0b 100644 --- a/test/reencodarr/ab_av1/worker_sessions_test.exs +++ b/test/reencodarr/ab_av1/worker_sessions_test.exs @@ -104,7 +104,8 @@ defmodule Reencodarr.AbAv1.WorkerSessionsTest do assert output =~ "worker-client-1" assert output =~ "protocol=1" assert output =~ "version=0.10.0" - assert output =~ "state=analyzed" + assert output =~ "phase=crf_searching" + assert output =~ "video_state=analyzed" end test "tracks an assigned video on the session" do @@ -143,8 +144,31 @@ defmodule Reencodarr.AbAv1.WorkerSessionsTest do assert {:ok, session} = WorkerSessions.finish_transfer("worker-server-1") assert session.active_video_id == 123 - assert session.phase == :crf_searching + assert session.phase == :input_ready + assert session.transfer_progress.percent == 100.0 + end + + test "rejects progress for a different active video" do + assert {:ok, _session} = WorkerSessions.register(worker_session_attrs()) + assert {:ok, _session} = WorkerSessions.assign_video("worker-server-1", 123, :receiving_input) + + assert {:error, :invalid_worker_phase} = + WorkerSessions.set_transfer_progress("worker-server-1", %{ + video_id: 456, + percent: 25.0 + }) + + assert {:error, :invalid_worker_phase} = + WorkerSessions.set_crf_search_progress("worker-server-1", %{ + video_id: 456, + percent: 25.0 + }) + + session = WorkerSessions.get("worker-server-1") + assert session.active_video_id == 123 + assert session.phase == :receiving_input assert is_nil(session.transfer_progress) + assert is_nil(session.crf_search_progress) end test "keeps CRF sample metadata when later progress omits it" do diff --git a/test/reencodarr_web/channels/worker_channel_test.exs b/test/reencodarr_web/channels/worker_channel_test.exs index de1f0871..a7a8a90b 100644 --- a/test/reencodarr_web/channels/worker_channel_test.exs +++ b/test/reencodarr_web/channels/worker_channel_test.exs @@ -93,6 +93,7 @@ defmodule ReencodarrWeb.WorkerChannelTest do assert Media.get_video(video.id).crf_search_worker_id == "worker-a" assert [session] = WorkerSessions.list() assert session.active_video_id == video.id + assert session.phase == :receiving_input assert is_nil(session.transfer_progress) after Application.delete_env(:reencodarr, :worker_token) @@ -411,7 +412,7 @@ defmodule ReencodarrWeb.WorkerChannelTest do :ok, %{accepted: true, event: "crf_search_progress"} - assert {:ok, _} = + assert {:error, :invalid_worker_phase} = WorkerSessions.set_transfer_progress(server_worker_id, %{ job_id: Integer.to_string(video_id), video_id: video_id, @@ -447,7 +448,7 @@ defmodule ReencodarrWeb.WorkerChannelTest do Application.delete_env(:reencodarr, :worker_token) end - test "fails orphaned dispatched work when reconnecting worker asks for new work" do + test "reattaches dispatched work when reconnecting worker asks for work" do token = "test-worker-token" Application.put_env(:reencodarr, :worker_token, token) @@ -469,13 +470,13 @@ defmodule ReencodarrWeb.WorkerChannelTest do :ok, %{accepted: true, protocol_version: 1} - assert_reply push(socket, "pull_work", %{}), :ok, %{status: "no_work"} + assert_reply push(socket, "pull_work", %{}), + :ok, + %{status: "job_in_progress", video_id: ^video_id} - assert Media.get_video(video_id).state == :failed - assert [failure] = Media.get_video_failures(video_id) - assert failure.failure_stage == :crf_search - assert failure.failure_category == :crf_optimization - assert failure.failure_code == "worker_disconnected" + assert Media.get_video(video_id).state == :crf_searching + assert Media.get_video_failures(video_id) == [] + assert WorkerSessions.get(socket.assigns.worker_id).phase == :crf_searching refute_push "transfer_started", _, 50 end) after @@ -586,7 +587,8 @@ defmodule ReencodarrWeb.WorkerChannelTest do capabilities: %{"crf_search" => true} }) - assert {:ok, _session} = WorkerSessions.assign_video("worker-server-1", video_id) + assert {:ok, _session} = + WorkerSessions.assign_video("worker-server-1", video_id, :receiving_input) assert {:ok, _session} = WorkerSessions.set_transfer_progress("worker-server-1", %{ @@ -630,7 +632,7 @@ defmodule ReencodarrWeb.WorkerChannelTest do Application.delete_env(:reencodarr, :worker_token) end - test "does not restart dispatched work when the session state is gone" do + test "reattaches dispatched work when the session state is gone" do token = "test-worker-token" Application.put_env(:reencodarr, :worker_token, token) @@ -656,9 +658,12 @@ defmodule ReencodarrWeb.WorkerChannelTest do :ok, %{accepted: true, protocol_version: 1} - assert_reply push(socket, "pull_work", %{}), :ok, %{status: "no_work"} + assert_reply push(socket, "pull_work", %{}), + :ok, + %{status: "job_in_progress", video_id: ^video_id} - assert Media.get_video(video_id).state == :failed + assert Media.get_video(video_id).state == :crf_searching + assert WorkerSessions.get(socket.assigns.worker_id).phase == :crf_searching refute_push "transfer_started", _, 50 end) after diff --git a/test/reencodarr_web/live/workers_live_test.exs b/test/reencodarr_web/live/workers_live_test.exs index 8581ef87..5f434a70 100644 --- a/test/reencodarr_web/live/workers_live_test.exs +++ b/test/reencodarr_web/live/workers_live_test.exs @@ -48,17 +48,6 @@ defmodule ReencodarrWeb.WorkersLiveTest do disk_total_bytes: 1_099_511_627_776 }) - assert {:ok, _session} = - WorkerSessions.set_crf_search_progress("worker-server-1", %{ - video_id: video.id, - percent: 62.0, - fps: 12.5, - eta: 90, - crf: 28.0, - sample_num: 3, - total_samples: 8 - }) - assert {:ok, _session} = WorkerSessions.set_transfer_progress("worker-server-1", %{ job_id: "job-1", @@ -87,9 +76,6 @@ defmodule ReencodarrWeb.WorkersLiveTest do assert html =~ "Chunk 3 / 8" assert html =~ "25.5%" assert html =~ "ETA 15s" - refute html =~ "Progress 62.0%" - refute html =~ "FPS 12.5 fps" - refute html =~ "Sample 3/8 - CRF 28.0" refute html =~ "CRF 28.0 -> 95.4 VMAF" end @@ -135,7 +121,7 @@ defmodule ReencodarrWeb.WorkersLiveTest do refute html =~ "Receiving Input" end - test "keeps the crf panel active after input transfer finishes", %{conn: conn} do + test "shows input ready after input transfer finishes before CRF progress", %{conn: conn} do {:ok, _session} = WorkerSessions.register(%{ server_worker_id: "worker-server-3", @@ -168,9 +154,9 @@ defmodule ReencodarrWeb.WorkersLiveTest do html = render(view) assert html =~ "worker-client-3" - assert html =~ "crf_searching" - assert html =~ "CRF Search" - refute html =~ "Receiving Input" + assert html =~ "Input ready" + assert html =~ "Input Ready" + refute html =~ "CRF Search" refute html =~ "-%" end end From 11551a118452f95b06c7bb84483de5195149f8d3 Mon Sep 17 00:00:00 2001 From: Mika Cohen Date: Wed, 8 Jul 2026 22:54:10 -0600 Subject: [PATCH 64/70] Backpressure worker input transfers --- lib/reencodarr_web/channels/worker_channel.ex | 29 +++++++++++++++-- .../channels/worker_channel_test.exs | 32 ++++++++++++++++++- 2 files changed, 57 insertions(+), 4 deletions(-) diff --git a/lib/reencodarr_web/channels/worker_channel.ex b/lib/reencodarr_web/channels/worker_channel.ex index ab75fc77..2e332bfe 100644 --- a/lib/reencodarr_web/channels/worker_channel.ex +++ b/lib/reencodarr_web/channels/worker_channel.ex @@ -277,6 +277,9 @@ defmodule ReencodarrWeb.WorkerChannel do :ok <- ensure_active_video(socket, progress.video_id) do _ = WorkerSessions.set_transfer_progress(worker_id, progress) Events.broadcast_event(:transfer_progress, Map.put(progress, :worker_id, worker_id)) + + socket = maybe_send_next_transfer_chunk(socket, progress) + {:reply, {:ok, WorkerProtocol.event_ack("transfer_progress")}, socket} else {:error, reason} -> @@ -405,6 +408,7 @@ defmodule ReencodarrWeb.WorkerChannel do |> assign(:transfer_total_chunks, total_chunks) |> assign(:transfer_bytes_sent, 0) |> assign(:transfer_chunk_index, 0) + |> assign(:transfer_waiting_for_ack, false) |> assign(:transfer_started_sent, nil) if File.exists?(video.path) do @@ -787,6 +791,7 @@ defmodule ReencodarrWeb.WorkerChannel do |> assign(:transfer_total_chunks, nil) |> assign(:transfer_bytes_sent, nil) |> assign(:transfer_chunk_index, nil) + |> assign(:transfer_waiting_for_ack, nil) |> assign(:transfer_started_sent, nil) end @@ -919,12 +924,11 @@ defmodule ReencodarrWeb.WorkerChannel do ) ) - send(self(), :stream_transfer_chunk) - {:noreply, socket |> assign(:transfer_bytes_sent, bytes_sent) - |> assign(:transfer_chunk_index, chunk_index + 1)} + |> assign(:transfer_chunk_index, chunk_index + 1) + |> assign(:transfer_waiting_for_ack, true)} :eof -> close_transfer_stream(io_device) @@ -950,6 +954,7 @@ defmodule ReencodarrWeb.WorkerChannel do |> assign(:transfer_total_chunks, nil) |> assign(:transfer_bytes_sent, nil) |> assign(:transfer_chunk_index, nil) + |> assign(:transfer_waiting_for_ack, nil) |> assign(:transfer_started_sent, nil)} {:error, reason} -> @@ -992,6 +997,24 @@ defmodule ReencodarrWeb.WorkerChannel do defp close_transfer_stream(nil), do: :ok defp close_transfer_stream(io_device), do: File.close(io_device) + defp maybe_send_next_transfer_chunk( + %{ + assigns: %{ + transfer_id: transfer_id, + transfer_bytes_sent: bytes_sent, + transfer_waiting_for_ack: true + } + } = socket, + %{transfer_id: transfer_id, bytes_sent: acknowledged_bytes} + ) + when is_integer(bytes_sent) and is_integer(acknowledged_bytes) and + acknowledged_bytes >= bytes_sent do + send(self(), :stream_transfer_chunk) + assign(socket, :transfer_waiting_for_ack, false) + end + + defp maybe_send_next_transfer_chunk(socket, _progress), do: socket + defp format_file_error(reason) do reason |> :file.format_error() |> List.to_string() end diff --git a/test/reencodarr_web/channels/worker_channel_test.exs b/test/reencodarr_web/channels/worker_channel_test.exs index a7a8a90b..7ddb1ffa 100644 --- a/test/reencodarr_web/channels/worker_channel_test.exs +++ b/test/reencodarr_web/channels/worker_channel_test.exs @@ -751,6 +751,21 @@ defmodule ReencodarrWeb.WorkerChannelTest do data: ^first_chunk }} = WorkerProtocol.parse_transfer_chunk_frame(first_frame) + refute_push "transfer_chunk", _, 50 + + assert_reply push(socket, "transfer_progress", %{ + "job_id" => transfer_id, + "transfer_id" => transfer_id, + "video_id" => assigned_video_id, + "percent" => 50.0, + "received_bytes" => chunk_size, + "expected_bytes" => content_size, + "chunk_index" => 0, + "total_chunks" => 2 + }), + :ok, + %{accepted: true, event: "transfer_progress"} + assert_push "transfer_chunk", {:binary, second_frame} assert {:ok, @@ -765,13 +780,28 @@ defmodule ReencodarrWeb.WorkerChannelTest do data: ^second_chunk }} = WorkerProtocol.parse_transfer_chunk_frame(second_frame) + refute_push "transfer_complete", _, 50 + + assert_reply push(socket, "transfer_progress", %{ + "job_id" => transfer_id, + "transfer_id" => transfer_id, + "video_id" => assigned_video_id, + "percent" => 100.0, + "received_bytes" => content_size, + "expected_bytes" => content_size, + "chunk_index" => 1, + "total_chunks" => 2 + }), + :ok, + %{accepted: true, event: "transfer_progress"} + session = WorkerSessions.get(socket.assigns.worker_id) assert session.active_video_id == assigned_video_id assert session.phase == :receiving_input assert session.transfer_progress.video_id == assigned_video_id assert session.transfer_progress.bytes_sent == content_size assert session.transfer_progress.total_bytes == content_size - assert session.transfer_progress.chunk_index == 2 + assert session.transfer_progress.chunk_index == 1 assert session.transfer_progress.total_chunks == 2 assert session.transfer_progress.percent == 100.0 From 75f71965ae3e31a182b1a7e84c507d48072d6d43 Mon Sep 17 00:00:00 2001 From: Mika Cohen Date: Wed, 8 Jul 2026 23:08:59 -0600 Subject: [PATCH 65/70] Gate worker transfer completion on client ack --- lib/reencodarr_web/channels/worker_channel.ex | 116 +++++++++--------- 1 file changed, 57 insertions(+), 59 deletions(-) diff --git a/lib/reencodarr_web/channels/worker_channel.ex b/lib/reencodarr_web/channels/worker_channel.ex index 2e332bfe..5be9f0ff 100644 --- a/lib/reencodarr_web/channels/worker_channel.ex +++ b/lib/reencodarr_web/channels/worker_channel.ex @@ -114,6 +114,18 @@ defmodule ReencodarrWeb.WorkerChannel do read_transfer_chunk(socket, io_device) end + def handle_info(:complete_transfer, %{assigns: %{transfer_io_device: io_device}} = socket) + when not is_nil(io_device) do + case Media.get_video(socket.assigns.current_video_id) do + %Media.Video{} = video -> + {:noreply, complete_transfer(socket, video, io_device)} + + nil -> + close_transfer_stream(io_device) + {:noreply, clear_assigned_video(socket.assigns.worker_id, socket)} + end + end + defp ensure_supported_protocol_version(protocol_version) do if WorkerProtocol.supported_protocol_version?(protocol_version) do :ok @@ -410,6 +422,7 @@ defmodule ReencodarrWeb.WorkerChannel do |> assign(:transfer_chunk_index, 0) |> assign(:transfer_waiting_for_ack, false) |> assign(:transfer_started_sent, nil) + |> assign(:transfer_complete_pending, nil) if File.exists?(video.path) do send(self(), :stream_transfer_chunk) @@ -793,6 +806,7 @@ defmodule ReencodarrWeb.WorkerChannel do |> assign(:transfer_chunk_index, nil) |> assign(:transfer_waiting_for_ack, nil) |> assign(:transfer_started_sent, nil) + |> assign(:transfer_complete_pending, nil) end defp mark_crf_search_active(worker_id, video_id) do @@ -809,6 +823,7 @@ defmodule ReencodarrWeb.WorkerChannel do socket |> assign(:transfer_io_device, io_device) |> assign(:transfer_started_sent, false) + |> assign(:transfer_complete_pending, false) {:ok, socket} @@ -867,31 +882,6 @@ defmodule ReencodarrWeb.WorkerChannel do } end - defp outbound_transfer_progress(socket, video, bytes_sent, chunk_index) do - total_bytes = socket.assigns.transfer_total_bytes - - %WorkerProtocol.TransferProgress{ - job_id: socket.assigns.transfer_id, - video_id: video.id, - transfer_id: socket.assigns.transfer_id, - filename: Path.basename(video.path), - percent: transfer_percent(bytes_sent, total_bytes), - bytes_sent: bytes_sent, - total_bytes: total_bytes, - bytes_per_second: nil, - eta: nil, - chunk_index: chunk_index, - total_chunks: socket.assigns.transfer_total_chunks - } - end - - defp transfer_percent(bytes_sent, total_bytes) - when is_integer(bytes_sent) and is_integer(total_bytes) and total_bytes > 0 do - bytes_sent / total_bytes * 100.0 - end - - defp transfer_percent(_bytes_sent, _total_bytes), do: 0.0 - defp read_transfer_chunk(%{assigns: %{current_video_id: video_id}} = socket, io_device) do video = Media.get_video(video_id) @@ -904,12 +894,6 @@ defmodule ReencodarrWeb.WorkerChannel do chunk_index = socket.assigns.transfer_chunk_index bytes_sent = socket.assigns.transfer_bytes_sent + byte_size(chunk) - _ = - WorkerSessions.set_transfer_progress( - socket.assigns.worker_id, - outbound_transfer_progress(socket, video, bytes_sent, chunk_index + 1) - ) - push( socket, "transfer_chunk", @@ -928,34 +912,14 @@ defmodule ReencodarrWeb.WorkerChannel do socket |> assign(:transfer_bytes_sent, bytes_sent) |> assign(:transfer_chunk_index, chunk_index + 1) - |> assign(:transfer_waiting_for_ack, true)} + |> assign(:transfer_waiting_for_ack, true) + |> assign( + :transfer_complete_pending, + bytes_sent >= socket.assigns.transfer_total_bytes + )} :eof -> - close_transfer_stream(io_device) - - push( - socket, - "transfer_complete", - WorkerProtocol.transfer_complete( - video, - socket.assigns.transfer_id, - socket.assigns.transfer_total_bytes, - socket.assigns.transfer_total_chunks - ) - ) - - {:noreply, - socket - |> assign(:transfer_io_device, nil) - |> assign(:transfer_path, nil) - |> assign(:transfer_id, nil) - |> assign(:transfer_chunk_size_bytes, nil) - |> assign(:transfer_total_bytes, nil) - |> assign(:transfer_total_chunks, nil) - |> assign(:transfer_bytes_sent, nil) - |> assign(:transfer_chunk_index, nil) - |> assign(:transfer_waiting_for_ack, nil) - |> assign(:transfer_started_sent, nil)} + {:noreply, complete_transfer(socket, video, io_device)} {:error, reason} -> close_transfer_stream(io_device) @@ -964,6 +928,34 @@ defmodule ReencodarrWeb.WorkerChannel do end end + defp complete_transfer(socket, video, io_device) do + close_transfer_stream(io_device) + + push( + socket, + "transfer_complete", + WorkerProtocol.transfer_complete( + video, + socket.assigns.transfer_id, + socket.assigns.transfer_total_bytes, + socket.assigns.transfer_total_chunks + ) + ) + + socket + |> assign(:transfer_io_device, nil) + |> assign(:transfer_path, nil) + |> assign(:transfer_id, nil) + |> assign(:transfer_chunk_size_bytes, nil) + |> assign(:transfer_total_bytes, nil) + |> assign(:transfer_total_chunks, nil) + |> assign(:transfer_bytes_sent, nil) + |> assign(:transfer_chunk_index, nil) + |> assign(:transfer_waiting_for_ack, nil) + |> assign(:transfer_started_sent, nil) + |> assign(:transfer_complete_pending, nil) + end + defp handle_transfer_failure(socket, video_id, reason) do case Media.get_video(video_id) do nil -> @@ -1002,14 +994,20 @@ defmodule ReencodarrWeb.WorkerChannel do assigns: %{ transfer_id: transfer_id, transfer_bytes_sent: bytes_sent, - transfer_waiting_for_ack: true + transfer_waiting_for_ack: true, + transfer_complete_pending: complete_pending? } } = socket, %{transfer_id: transfer_id, bytes_sent: acknowledged_bytes} ) when is_integer(bytes_sent) and is_integer(acknowledged_bytes) and acknowledged_bytes >= bytes_sent do - send(self(), :stream_transfer_chunk) + if complete_pending? do + send(self(), :complete_transfer) + else + send(self(), :stream_transfer_chunk) + end + assign(socket, :transfer_waiting_for_ack, false) end From 59aa185f16563a5000889ef3c3f515036a215a66 Mon Sep 17 00:00:00 2001 From: Mika Cohen Date: Wed, 8 Jul 2026 23:24:28 -0600 Subject: [PATCH 66/70] Derive worker transfer rate on server --- lib/reencodarr_web/channels/worker_channel.ex | 65 +++++++++++++++++++ .../channels/worker_channel_test.exs | 29 +++++++++ 2 files changed, 94 insertions(+) diff --git a/lib/reencodarr_web/channels/worker_channel.ex b/lib/reencodarr_web/channels/worker_channel.ex index 5be9f0ff..e2a0b410 100644 --- a/lib/reencodarr_web/channels/worker_channel.ex +++ b/lib/reencodarr_web/channels/worker_channel.ex @@ -287,6 +287,8 @@ defmodule ReencodarrWeb.WorkerChannel do defp handle_transfer_progress(payload, %{assigns: %{worker_id: worker_id}} = socket) do with {:ok, progress} <- WorkerProtocol.parse_transfer_progress(payload), :ok <- ensure_active_video(socket, progress.video_id) do + {progress, socket} = fill_transfer_rate(socket, progress) + _ = WorkerSessions.set_transfer_progress(worker_id, progress) Events.broadcast_event(:transfer_progress, Map.put(progress, :worker_id, worker_id)) @@ -423,6 +425,8 @@ defmodule ReencodarrWeb.WorkerChannel do |> assign(:transfer_waiting_for_ack, false) |> assign(:transfer_started_sent, nil) |> assign(:transfer_complete_pending, nil) + |> assign(:transfer_last_progress_at, nil) + |> assign(:transfer_last_progress_bytes, nil) if File.exists?(video.path) do send(self(), :stream_transfer_chunk) @@ -807,6 +811,8 @@ defmodule ReencodarrWeb.WorkerChannel do |> assign(:transfer_waiting_for_ack, nil) |> assign(:transfer_started_sent, nil) |> assign(:transfer_complete_pending, nil) + |> assign(:transfer_last_progress_at, nil) + |> assign(:transfer_last_progress_bytes, nil) end defp mark_crf_search_active(worker_id, video_id) do @@ -824,6 +830,8 @@ defmodule ReencodarrWeb.WorkerChannel do |> assign(:transfer_io_device, io_device) |> assign(:transfer_started_sent, false) |> assign(:transfer_complete_pending, false) + |> assign(:transfer_last_progress_at, nil) + |> assign(:transfer_last_progress_bytes, nil) {:ok, socket} @@ -954,8 +962,65 @@ defmodule ReencodarrWeb.WorkerChannel do |> assign(:transfer_waiting_for_ack, nil) |> assign(:transfer_started_sent, nil) |> assign(:transfer_complete_pending, nil) + |> assign(:transfer_last_progress_at, nil) + |> assign(:transfer_last_progress_bytes, nil) + end + + defp fill_transfer_rate(socket, progress) do + now = System.monotonic_time(:millisecond) + + progress = + progress + |> maybe_fill_bytes_per_second(socket, now) + |> maybe_fill_eta() + + socket = + socket + |> assign(:transfer_last_progress_at, now) + |> assign(:transfer_last_progress_bytes, progress.bytes_sent) + + {progress, socket} end + defp maybe_fill_bytes_per_second(%{bytes_per_second: rate} = progress, _socket, _now) + when is_integer(rate) and rate >= 0, + do: progress + + defp maybe_fill_bytes_per_second( + %{bytes_sent: bytes_sent} = progress, + %{ + assigns: %{ + transfer_last_progress_at: last_at, + transfer_last_progress_bytes: last_bytes + } + }, + now + ) + when is_integer(bytes_sent) and is_integer(last_bytes) and is_integer(last_at) and + bytes_sent >= last_bytes and now > last_at do + elapsed_ms = now - last_at + bytes_delta = bytes_sent - last_bytes + + if bytes_delta > 0 do + %{progress | bytes_per_second: div(bytes_delta * 1_000, elapsed_ms)} + else + progress + end + end + + defp maybe_fill_bytes_per_second(progress, _socket, _now), do: progress + + defp maybe_fill_eta(%{eta: eta} = progress) when is_integer(eta) and eta >= 0, + do: progress + + defp maybe_fill_eta(%{bytes_per_second: rate, bytes_sent: sent, total_bytes: total} = progress) + when is_integer(rate) and rate > 0 and is_integer(sent) and is_integer(total) and + total > sent do + %{progress | eta: ceil((total - sent) / rate)} + end + + defp maybe_fill_eta(progress), do: progress + defp handle_transfer_failure(socket, video_id, reason) do case Media.get_video(video_id) do nil -> diff --git a/test/reencodarr_web/channels/worker_channel_test.exs b/test/reencodarr_web/channels/worker_channel_test.exs index 7ddb1ffa..7113256d 100644 --- a/test/reencodarr_web/channels/worker_channel_test.exs +++ b/test/reencodarr_web/channels/worker_channel_test.exs @@ -301,6 +301,35 @@ defmodule ReencodarrWeb.WorkerChannelTest do assert filename == Path.basename(video.path) + Process.sleep(10) + + assert_reply push(socket, "transfer_progress", %{ + "video_id" => video_id, + "job_id" => "job-1", + "filename" => Path.basename(video.path), + "percent" => 50.0, + "received_bytes" => 5_242_880, + "expected_bytes" => 10_485_760, + "chunk_index" => 3, + "total_chunks" => 8 + }), + :ok, + %{accepted: true, event: "transfer_progress"} + + assert_receive {:transfer_progress, + %{ + video_id: ^video_id, + bytes_sent: 5_242_880, + total_bytes: 10_485_760, + bytes_per_second: derived_rate, + eta: derived_eta + }} + + assert is_integer(derived_rate) + assert derived_rate > 0 + assert is_integer(derived_eta) + assert derived_eta >= 0 + assert_reply push(socket, "crf_search_progress", %{ "video_id" => video_id, "percent" => 62.0, From 92423eba90e8bd40e9da95beb49c45a140eca081 Mon Sep 17 00:00:00 2001 From: Mika Cohen Date: Thu, 9 Jul 2026 09:04:25 -0600 Subject: [PATCH 67/70] Persist worker VMAFs before completion --- lib/reencodarr/ab_av1/worker_protocol.ex | 15 +++- lib/reencodarr_web/channels/worker_channel.ex | 76 ++++++++++++++----- .../channels/worker_channel_test.exs | 58 ++++++++++++++ 3 files changed, 126 insertions(+), 23 deletions(-) diff --git a/lib/reencodarr/ab_av1/worker_protocol.ex b/lib/reencodarr/ab_av1/worker_protocol.ex index 8aa0bfe5..9531bbf6 100644 --- a/lib/reencodarr/ab_av1/worker_protocol.ex +++ b/lib/reencodarr/ab_av1/worker_protocol.ex @@ -121,12 +121,13 @@ defmodule Reencodarr.AbAv1.WorkerProtocol do @moduledoc false @enforce_keys [:video_id, :result] - defstruct [:video_id, :result, :chosen_crf] + defstruct [:video_id, :result, :chosen_crf, results: []] @type t :: %__MODULE__{ video_id: pos_integer(), result: :ok | :cancelled | :shutdown | :failed | {:error, term()}, - chosen_crf: number() | nil + chosen_crf: number() | nil, + results: [map()] } end @@ -296,7 +297,8 @@ defmodule Reencodarr.AbAv1.WorkerProtocol do %Completion{ video_id: video_id, result: result, - chosen_crf: optional_number(payload, [:chosen_crf, "chosen_crf"]) + chosen_crf: optional_number(payload, [:chosen_crf, "chosen_crf"]), + results: parse_optional_completion_results(payload) }} end end @@ -618,6 +620,13 @@ defmodule Reencodarr.AbAv1.WorkerProtocol do end end + defp parse_optional_completion_results(payload) do + case parse_result_batch(payload) do + {:ok, results} -> results + {:error, _reason} -> [] + end + end + defp parse_result_list(results) do results |> Enum.reduce_while({:ok, []}, fn result, {:ok, acc} -> diff --git a/lib/reencodarr_web/channels/worker_channel.ex b/lib/reencodarr_web/channels/worker_channel.ex index e2a0b410..98f487a0 100644 --- a/lib/reencodarr_web/channels/worker_channel.ex +++ b/lib/reencodarr_web/channels/worker_channel.ex @@ -491,16 +491,30 @@ defmodule ReencodarrWeb.WorkerChannel do video -> mark_crf_search_active(socket.assigns.worker_id, video_id) - persist_crf_results(video, results) - if Enum.any?(results, &Map.get(&1, :chosen, false)) do - finalize_chosen_result_from_report(socket.assigns.worker_id, socket, video, results) - else - {:reply, {:ok, WorkerProtocol.event_ack("crf_search_result")}, socket} + case persist_crf_results(video, results) do + {:ok, persisted_results} -> + reply_after_persisted_crf_results(socket, video, persisted_results) + + {:error, reason} -> + {:reply, {:error, WorkerProtocol.error(reason)}, socket} end end end + defp reply_after_persisted_crf_results(socket, video, persisted_results) do + if Enum.any?(persisted_results, &Map.get(&1, :chosen, false)) do + finalize_chosen_result_from_report( + socket.assigns.worker_id, + socket, + video, + persisted_results + ) + else + {:reply, {:ok, WorkerProtocol.event_ack("crf_search_result")}, socket} + end + end + defp handle_parsed_crf_search_result(%{assigns: %{worker_id: worker_id}} = socket, result) do case Media.get_video(result.video_id) do %Media.Video{state: :crf_searched, chosen_vmaf_id: chosen_vmaf_id} @@ -537,7 +551,8 @@ defmodule ReencodarrWeb.WorkerChannel do socket, video, completion.result, - completion.chosen_crf + completion.chosen_crf, + completion.results ) do {:ok, socket} -> Events.broadcast_event(:crf_search_completed, %{ @@ -572,23 +587,30 @@ defmodule ReencodarrWeb.WorkerChannel do end end - defp apply_completion_result(worker_id, socket, video, :ok, chosen_crf) do - finish_successful_crf_search(worker_id, socket, video, chosen_crf) + defp apply_completion_result(worker_id, socket, video, :ok, chosen_crf, results) do + case persist_crf_results(video, results) do + {:ok, persisted_results} -> + chosen_crf = chosen_crf || chosen_crf_from_results(persisted_results) + finish_successful_crf_search(worker_id, socket, video, chosen_crf) + + {:error, reason} -> + fail_invalid_successful_crf_search(worker_id, socket, video, reason) + end end - defp apply_completion_result(worker_id, socket, video, :cancelled, _chosen_crf) do + defp apply_completion_result(worker_id, socket, video, :cancelled, _chosen_crf, _results) do finish_cancelled_crf_search(worker_id, socket, video, :cancelled) end - defp apply_completion_result(worker_id, socket, video, :shutdown, _chosen_crf) do + defp apply_completion_result(worker_id, socket, video, :shutdown, _chosen_crf, _results) do finish_cancelled_crf_search(worker_id, socket, video, :shutdown) end - defp apply_completion_result(worker_id, socket, video, :failed, _chosen_crf) do + defp apply_completion_result(worker_id, socket, video, :failed, _chosen_crf, _results) do record_completion_failure(worker_id, socket, video, "failed") end - defp apply_completion_result(worker_id, socket, video, {:error, reason}, _chosen_crf) do + defp apply_completion_result(worker_id, socket, video, {:error, reason}, _chosen_crf, _results) do record_completion_failure(worker_id, socket, video, inspect(reason)) end @@ -680,34 +702,42 @@ defmodule ReencodarrWeb.WorkerChannel do do: socket.assigns[:client_worker_id] || socket.assigns.worker_id defp persist_crf_results(video, results) do - Enum.each(results, fn result -> + Enum.reduce_while(results, {:ok, []}, fn result, {:ok, acc} -> params = CrfSearch.build_crf_search_args(video, result_target(video, result)) attrs = result |> Map.put(:video_id, video.id) |> Map.update(:params, params, fn + [] -> params nil -> params existing_params -> existing_params end) |> Map.delete(:chosen) case Media.upsert_vmaf(attrs) do - {:ok, :skipped} -> :ok - {:ok, vmaf} -> Events.broadcast_event(:crf_search_vmaf_result, vmaf_to_event(vmaf)) - {:error, _} -> :ok + {:ok, :skipped} -> + {:cont, {:ok, [result | acc]}} + + {:ok, vmaf} -> + Events.broadcast_event(:crf_search_vmaf_result, vmaf_to_event(vmaf)) + {:cont, {:ok, [result | acc]}} + + {:error, _reason} -> + {:halt, {:error, :invalid_crf_search_result}} end end) + |> case do + {:ok, persisted_results} -> {:ok, Enum.reverse(persisted_results)} + {:error, _reason} = error -> error + end end defp result_target(video, result), do: Map.get(result, :target) || Reencodarr.Rules.vmaf_target(video) defp finalize_chosen_result_from_report(worker_id, socket, video, results) do - chosen_crf = - Enum.find_value(results, fn result -> - if Map.get(result, :chosen, false), do: Map.get(result, :crf) - end) + chosen_crf = chosen_crf_from_results(results) case chosen_crf do nil -> @@ -718,6 +748,12 @@ defmodule ReencodarrWeb.WorkerChannel do end end + defp chosen_crf_from_results(results) do + Enum.find_value(results, fn result -> + if Map.get(result, :chosen, false), do: Map.get(result, :crf) + end) + end + defp mark_reported_chosen_crf(worker_id, socket, video, crf) do case Media.mark_vmaf_as_chosen(video.id, crf) do {:ok, _} -> diff --git a/test/reencodarr_web/channels/worker_channel_test.exs b/test/reencodarr_web/channels/worker_channel_test.exs index 7113256d..dd398ca9 100644 --- a/test/reencodarr_web/channels/worker_channel_test.exs +++ b/test/reencodarr_web/channels/worker_channel_test.exs @@ -1014,6 +1014,64 @@ defmodule ReencodarrWeb.WorkerChannelTest do Application.delete_env(:reencodarr, :worker_token) end + test "persists completion VMAF results before completing the job" do + token = "test-worker-token" + Application.put_env(:reencodarr, :worker_token, token) + Phoenix.PubSub.subscribe(Reencodarr.PubSub, Events.channel()) + + {:ok, video} = Fixtures.video_fixture(%{state: :analyzed}) + video_id = video.id + + assert {:ok, socket} = connect(WorkerSocket, %{"token" => token}) + assert {:ok, _join_payload, socket} = subscribe_and_join(socket, "workers:crf_search") + + assert_reply push(socket, "announce", announce_payload(worker_id: "worker-a")), + :ok, + %{accepted: true, protocol_version: 1} + + assert_reply push(socket, "pull_work", %{}), + :ok, + %{status: "job_assigned", video_id: ^video_id} + + assert_reply push(socket, "crf_search_completed", %{ + "video_id" => video_id, + "result" => "ok", + "results" => [ + %{ + "crf" => 13, + "vmaf_score" => 90.74086, + "encode_percent" => 108.0, + "predicted_encode_size" => 5_230_000_000, + "predicted_encode_time_secs" => 5160 + }, + %{ + "crf" => 12, + "vmaf_score" => 94.86, + "encode_percent" => 8.0, + "predicted_encode_size" => 263_590_000, + "predicted_encode_time_secs" => 1260, + "chosen" => true + } + ] + }), + :ok, + %{accepted: true, event: "crf_search_completed"} + + assert_receive {:crf_search_vmaf_result, %{video_id: ^video_id, crf: 13.0, score: 90.74086}} + assert_receive {:crf_search_vmaf_result, %{video_id: ^video_id, crf: 12.0, score: 94.86}} + + vmafs = Media.get_vmafs_for_video(video_id) + assert Enum.any?(vmafs, &(&1.crf == 13.0 and &1.score == 90.74086)) + assert Enum.any?(vmafs, &(&1.crf == 12.0 and &1.score == 94.86)) + + video = Media.get_video(video_id) + assert video.state == :crf_searched + assert video.chosen_vmaf_id != nil + assert WorkerSessions.get(socket.assigns.worker_id).active_video_id == nil + after + Application.delete_env(:reencodarr, :worker_token) + end + test "fails a completed CRF search when nothing was chosen" do token = "test-worker-token" Application.put_env(:reencodarr, :worker_token, token) From ee6d3f303b922601ec79d5f7d00e91bde9592185 Mon Sep 17 00:00:00 2001 From: Mika Cohen Date: Thu, 9 Jul 2026 11:06:41 -0600 Subject: [PATCH 68/70] Add HTTP worker file transfer --- config/config.exs | 4 +- config/runtime.exs | 24 ++++++--- lib/reencodarr/ab_av1/worker_config.ex | 14 ++++++ lib/reencodarr/ab_av1/worker_protocol.ex | 26 ++++++++-- lib/reencodarr_web/channels/worker_channel.ex | 15 ++++-- .../controllers/worker_file_controller.ex | 33 +++++++++++++ lib/reencodarr_web/router.ex | 4 ++ .../ab_av1/worker_protocol_test.exs | 36 +++++++++++++- .../worker_file_controller_test.exs | 49 +++++++++++++++++++ 9 files changed, 188 insertions(+), 17 deletions(-) create mode 100644 lib/reencodarr_web/controllers/worker_file_controller.ex create mode 100644 test/reencodarr_web/controllers/worker_file_controller_test.exs diff --git a/config/config.exs b/config/config.exs index 164455f7..d5d7ba30 100644 --- a/config/config.exs +++ b/config/config.exs @@ -18,7 +18,9 @@ config :reencodarr, worker_transfer_window: 1, worker_retry_limit: 3, worker_transfer_timeout_ms: 60_000, - worker_max_concurrent_transfers: 1 + worker_max_concurrent_transfers: 1, + worker_transfer_base_url: nil, + worker_transfer_token: nil # Shared database configuration - SQLite performance tuning config :reencodarr, Reencodarr.Repo, diff --git a/config/runtime.exs b/config/runtime.exs index da15f2ee..ad8ce38f 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -34,8 +34,8 @@ parse_int_env = fn name, default -> end end -worker_token = - System.get_env("REENCODARR_WORKER_TOKEN") +derive_token = fn env_name, label -> + System.get_env(env_name) |> case do nil -> case System.get_env("SECRET_KEY_BASE") do @@ -43,18 +43,28 @@ worker_token = nil secret_key_base -> - :crypto.mac(:hmac, :sha256, secret_key_base, "ab-av1-worker-token") + :crypto.mac(:hmac, :sha256, secret_key_base, label) |> Base.url_encode64(padding: false) end - worker_token -> - worker_token + token -> + token end +end + +worker_token = derive_token.("REENCODARR_WORKER_TOKEN", "ab-av1-worker-token") + +worker_transfer_token = + derive_token.("REENCODARR_WORKER_TRANSFER_TOKEN", "ab-av1-worker-transfer-token") if worker_token do config :reencodarr, :worker_token, worker_token end +if worker_transfer_token do + config :reencodarr, :worker_transfer_token, worker_transfer_token +end + config :reencodarr, distributed_worker_enabled: parse_bool_env.(System.get_env("REENCODARR_DISTRIBUTED_WORKERS", "true")), @@ -62,7 +72,9 @@ config :reencodarr, worker_transfer_window: parse_int_env.("REENCODARR_WORKER_TRANSFER_WINDOW", 1), worker_retry_limit: parse_int_env.("REENCODARR_WORKER_RETRY_LIMIT", 3), worker_transfer_timeout_ms: parse_int_env.("REENCODARR_WORKER_TRANSFER_TIMEOUT_MS", 60_000), - worker_max_concurrent_transfers: parse_int_env.("REENCODARR_WORKER_MAX_CONCURRENT_TRANSFERS", 1) + worker_max_concurrent_transfers: + parse_int_env.("REENCODARR_WORKER_MAX_CONCURRENT_TRANSFERS", 1), + worker_transfer_base_url: System.get_env("REENCODARR_WORKER_TRANSFER_BASE_URL") if config_env() == :prod do # Parse CHECK_ORIGIN env var: "false" disables, otherwise comma-separated origin list diff --git a/lib/reencodarr/ab_av1/worker_config.ex b/lib/reencodarr/ab_av1/worker_config.ex index 60525940..f004c665 100644 --- a/lib/reencodarr/ab_av1/worker_config.ex +++ b/lib/reencodarr/ab_av1/worker_config.ex @@ -46,4 +46,18 @@ defmodule Reencodarr.AbAv1.WorkerConfig do def max_concurrent_transfers do Application.get_env(:reencodarr, :worker_max_concurrent_transfers, 1) end + + @spec transfer_base_url() :: String.t() | nil + def transfer_base_url do + case Application.get_env(:reencodarr, :worker_transfer_base_url) do + url when is_binary(url) and url != "" -> String.trim_trailing(url, "/") + _ -> nil + end + end + + @spec transfer_token() :: String.t() | nil + def transfer_token do + Application.get_env(:reencodarr, :worker_transfer_token) || + Application.get_env(:reencodarr, :worker_token) + end end diff --git a/lib/reencodarr/ab_av1/worker_protocol.ex b/lib/reencodarr/ab_av1/worker_protocol.ex index 9531bbf6..4c5a288d 100644 --- a/lib/reencodarr/ab_av1/worker_protocol.ex +++ b/lib/reencodarr/ab_av1/worker_protocol.ex @@ -462,7 +462,8 @@ defmodule Reencodarr.AbAv1.WorkerProtocol do @spec work_assigned(Video.t(), number()) :: map() def work_assigned(%Video{id: video_id, path: path, size: size} = video, target_vmaf) when is_integer(video_id) and is_binary(path) do - %{ + video + |> maybe_put_transfer(%{ status: "job_assigned", job_id: Integer.to_string(video_id), video_id: video_id, @@ -471,12 +472,13 @@ defmodule Reencodarr.AbAv1.WorkerProtocol do chunk_size_bytes: chunk_size_bytes(), target_vmaf: target_vmaf, crf_search_args: CrfSearch.build_crf_search_args(video, target_vmaf) - } + }) end def work_in_progress(%Video{id: video_id, path: path, size: size} = video, target_vmaf) when is_integer(video_id) and is_binary(path) do - %{ + video + |> maybe_put_transfer(%{ status: "job_in_progress", job_id: Integer.to_string(video_id), video_id: video_id, @@ -484,7 +486,23 @@ defmodule Reencodarr.AbAv1.WorkerProtocol do size_bytes: size || 0, target_vmaf: target_vmaf, crf_search_args: CrfSearch.build_crf_search_args(video, target_vmaf) - } + }) + end + + defp maybe_put_transfer(%Video{id: video_id}, payload) do + with base_url when is_binary(base_url) <- WorkerConfig.transfer_base_url(), + token when is_binary(token) <- WorkerConfig.transfer_token() do + Map.put(payload, :transfer, %{ + url: "#{base_url}/workers/files/#{video_id}", + auth: %{ + scheme: "bearer", + header: "authorization", + value: "Bearer #{token}" + } + }) + else + _ -> payload + end end @spec heartbeat_ack(DateTime.t()) :: map() diff --git a/lib/reencodarr_web/channels/worker_channel.ex b/lib/reencodarr_web/channels/worker_channel.ex index 98f487a0..275c726b 100644 --- a/lib/reencodarr_web/channels/worker_channel.ex +++ b/lib/reencodarr_web/channels/worker_channel.ex @@ -5,7 +5,7 @@ defmodule ReencodarrWeb.WorkerChannel do use ReencodarrWeb, :channel - alias Reencodarr.AbAv1.{CrfSearch, WorkerProtocol, WorkerSessions} + alias Reencodarr.AbAv1.{CrfSearch, WorkerConfig, WorkerProtocol, WorkerSessions} alias Reencodarr.AbAv1.WorkerProtocol.Announcement alias Reencodarr.Dashboard.Events alias Reencodarr.Media @@ -373,7 +373,9 @@ defmodule ReencodarrWeb.WorkerChannel do with {:ok, _video} <- Media.mark_as_worker_crf_searching(video, worker_dispatch_id(socket)), {:ok, _session} <- WorkerSessions.assign_video(worker_id, video.id, :receiving_input) do - socket = prepare_transfer(socket, video, target_vmaf) + socket = + prepare_transfer(socket, video, target_vmaf, stream?: websocket_transfer_on_assign?()) + {:reply, {:ok, WorkerProtocol.work_assigned(video, target_vmaf)}, socket} else {:error, reason} -> @@ -387,7 +389,7 @@ defmodule ReencodarrWeb.WorkerChannel do case WorkerSessions.assign_video(worker_id, video.id, :receiving_input) do {:ok, _session} -> - socket = prepare_transfer(socket, video, target_vmaf) + socket = prepare_transfer(socket, video, target_vmaf, stream?: true) {:reply, {:ok, WorkerProtocol.work_assigned(video, target_vmaf)}, socket} {:error, reason} -> @@ -404,7 +406,7 @@ defmodule ReencodarrWeb.WorkerChannel do socket} end - defp prepare_transfer(socket, video, target_vmaf) do + defp prepare_transfer(socket, video, target_vmaf, opts) do transfer_id = Integer.to_string(video.id) total_bytes = video.size || 0 chunk_size_bytes = WorkerProtocol.chunk_size_bytes() @@ -428,13 +430,16 @@ defmodule ReencodarrWeb.WorkerChannel do |> assign(:transfer_last_progress_at, nil) |> assign(:transfer_last_progress_bytes, nil) - if File.exists?(video.path) do + if Keyword.get(opts, :stream?, true) and File.exists?(video.path) do send(self(), :stream_transfer_chunk) end socket end + defp websocket_transfer_on_assign?, + do: is_nil(WorkerConfig.transfer_base_url()) + defp work_request_mode(payload) when is_map(payload) do if Enum.any?( [ diff --git a/lib/reencodarr_web/controllers/worker_file_controller.ex b/lib/reencodarr_web/controllers/worker_file_controller.ex new file mode 100644 index 00000000..2e5b6d3b --- /dev/null +++ b/lib/reencodarr_web/controllers/worker_file_controller.ex @@ -0,0 +1,33 @@ +defmodule ReencodarrWeb.WorkerFileController do + @moduledoc false + + use ReencodarrWeb, :controller + + alias Reencodarr.AbAv1.WorkerConfig + alias Reencodarr.Media + + def show(conn, %{"id" => id}) do + with :ok <- authorize(conn), + {video_id, ""} <- Integer.parse(id), + %Media.Video{path: path} <- Media.get_video(video_id), + true <- File.regular?(path) do + conn + |> put_resp_content_type("application/octet-stream") + |> send_file(200, path) + else + {:error, :unauthorized} -> send_resp(conn, 401, "unauthorized") + _ -> send_resp(conn, 404, "not found") + end + end + + defp authorize(conn) do + with ["Bearer " <> provided] <- get_req_header(conn, "authorization"), + token when is_binary(token) <- WorkerConfig.transfer_token(), + true <- byte_size(provided) == byte_size(token), + true <- Plug.Crypto.secure_compare(provided, token) do + :ok + else + _ -> {:error, :unauthorized} + end + end +end diff --git a/lib/reencodarr_web/router.ex b/lib/reencodarr_web/router.ex index 94050d76..4326f337 100644 --- a/lib/reencodarr_web/router.ex +++ b/lib/reencodarr_web/router.ex @@ -61,6 +61,10 @@ defmodule ReencodarrWeb.Router do post "/webhooks/radarr", RadarrWebhookController, :radarr end + scope "/workers", ReencodarrWeb do + get "/files/:id", WorkerFileController, :show + end + # Other scopes may use custom stacks. # scope "/api", ReencodarrWeb do # pipe_through :api diff --git a/test/reencodarr/ab_av1/worker_protocol_test.exs b/test/reencodarr/ab_av1/worker_protocol_test.exs index 14730669..75d82418 100644 --- a/test/reencodarr/ab_av1/worker_protocol_test.exs +++ b/test/reencodarr/ab_av1/worker_protocol_test.exs @@ -1,5 +1,5 @@ defmodule Reencodarr.AbAv1.WorkerProtocolTest do - use ExUnit.Case, async: true + use ExUnit.Case, async: false alias Reencodarr.AbAv1.WorkerProtocol alias Reencodarr.AbAv1.WorkerProtocol.Announcement @@ -251,6 +251,40 @@ defmodule Reencodarr.AbAv1.WorkerProtocolTest do assert "--temp-dir" in crf_search_args end + test "includes configured worker transfer URL in job payloads" do + previous_base_url = Application.get_env(:reencodarr, :worker_transfer_base_url) + previous_token = Application.get_env(:reencodarr, :worker_transfer_token) + Application.put_env(:reencodarr, :worker_transfer_base_url, "http://10.0.0.10:4000/") + Application.put_env(:reencodarr, :worker_transfer_token, "transfer-token") + + on_exit(fn -> + restore_env(:worker_transfer_base_url, previous_base_url) + restore_env(:worker_transfer_token, previous_token) + end) + + video = %Reencodarr.Media.Video{id: 123, path: "/videos/movie.mkv", size: 987_654} + + assert %{transfer: transfer} = WorkerProtocol.work_assigned(video, 96.5) + + assert transfer == %{ + url: "http://10.0.0.10:4000/workers/files/123", + auth: %{ + scheme: "bearer", + header: "authorization", + value: "Bearer transfer-token" + } + } + + assert %{transfer: ^transfer} = + WorkerProtocol.work_assigned(video, 96.5) + + assert %{transfer: ^transfer} = + WorkerProtocol.work_in_progress(video, 96.5) + end + + defp restore_env(key, nil), do: Application.delete_env(:reencodarr, key) + defp restore_env(key, value), do: Application.put_env(:reencodarr, key, value) + test "builds binary transfer chunk frames with ordered metadata and raw data" do chunk = "raw video bytes" diff --git a/test/reencodarr_web/controllers/worker_file_controller_test.exs b/test/reencodarr_web/controllers/worker_file_controller_test.exs new file mode 100644 index 00000000..c72dae1f --- /dev/null +++ b/test/reencodarr_web/controllers/worker_file_controller_test.exs @@ -0,0 +1,49 @@ +defmodule ReencodarrWeb.WorkerFileControllerTest do + use ReencodarrWeb.ConnCase, async: false + + test "requires worker bearer token", %{conn: conn} do + previous = Application.get_env(:reencodarr, :worker_token) + previous_transfer = Application.get_env(:reencodarr, :worker_transfer_token) + Application.put_env(:reencodarr, :worker_token, "test-token") + Application.delete_env(:reencodarr, :worker_transfer_token) + + on_exit(fn -> + restore_env(:worker_token, previous) + restore_env(:worker_transfer_token, previous_transfer) + end) + + conn = get(conn, ~p"/workers/files/123") + + assert conn.status == 401 + end + + test "sends the video file to authorized workers", %{conn: conn} do + previous = Application.get_env(:reencodarr, :worker_token) + previous_transfer = Application.get_env(:reencodarr, :worker_transfer_token) + Application.put_env(:reencodarr, :worker_transfer_token, "transfer-token") + + on_exit(fn -> + restore_env(:worker_token, previous) + restore_env(:worker_transfer_token, previous_transfer) + end) + + path = Path.join(System.tmp_dir!(), "reencodarr-worker-file-#{System.unique_integer()}.mkv") + File.write!(path, "video bytes") + on_exit(fn -> File.rm(path) end) + + {:ok, video} = Fixtures.video_fixture(%{path: path, size: byte_size("video bytes")}) + + conn = + conn + |> put_req_header("authorization", "Bearer transfer-token") + |> get(~p"/workers/files/#{video.id}") + + assert conn.status == 200 + assert conn.resp_body == "video bytes" + assert [content_type] = get_resp_header(conn, "content-type") + assert String.starts_with?(content_type, "application/octet-stream") + end + + defp restore_env(key, nil), do: Application.delete_env(:reencodarr, key) + defp restore_env(key, value), do: Application.put_env(:reencodarr, key, value) +end From 455565a9f5909802f8e3c6f70701b9aced9d4f81 Mon Sep 17 00:00:00 2001 From: Mika Cohen Date: Thu, 9 Jul 2026 11:27:14 -0600 Subject: [PATCH 69/70] Show HTTP worker transfer progress --- lib/reencodarr_web/channels/worker_channel.ex | 14 +++++++ lib/reencodarr_web/live/workers_live.ex | 11 +++-- .../channels/worker_channel_test.exs | 4 +- .../reencodarr_web/live/workers_live_test.exs | 40 +++++++++++++++++++ 4 files changed, 61 insertions(+), 8 deletions(-) diff --git a/lib/reencodarr_web/channels/worker_channel.ex b/lib/reencodarr_web/channels/worker_channel.ex index 275c726b..d61c7648 100644 --- a/lib/reencodarr_web/channels/worker_channel.ex +++ b/lib/reencodarr_web/channels/worker_channel.ex @@ -290,6 +290,7 @@ defmodule ReencodarrWeb.WorkerChannel do {progress, socket} = fill_transfer_rate(socket, progress) _ = WorkerSessions.set_transfer_progress(worker_id, progress) + _ = maybe_finish_transfer(worker_id, progress) Events.broadcast_event(:transfer_progress, Map.put(progress, :worker_id, worker_id)) socket = maybe_send_next_transfer_chunk(socket, progress) @@ -1119,6 +1120,19 @@ defmodule ReencodarrWeb.WorkerChannel do defp maybe_send_next_transfer_chunk(socket, _progress), do: socket + defp maybe_finish_transfer(worker_id, %{bytes_sent: bytes_sent, total_bytes: total_bytes}) + when is_integer(bytes_sent) and is_integer(total_bytes) and total_bytes > 0 and + bytes_sent >= total_bytes do + WorkerSessions.finish_transfer(worker_id) + end + + defp maybe_finish_transfer(worker_id, %{percent: percent}) + when is_number(percent) and percent >= 100 do + WorkerSessions.finish_transfer(worker_id) + end + + defp maybe_finish_transfer(_worker_id, _progress), do: :ok + defp format_file_error(reason) do reason |> :file.format_error() |> List.to_string() end diff --git a/lib/reencodarr_web/live/workers_live.ex b/lib/reencodarr_web/live/workers_live.ex index d46adc8b..62eb7e8a 100644 --- a/lib/reencodarr_web/live/workers_live.ex +++ b/lib/reencodarr_web/live/workers_live.ex @@ -211,7 +211,9 @@ defmodule ReencodarrWeb.WorkersLive do
{format_transfer_bytes(@worker.transfer_progress)} - Chunk {format_chunk_progress(@worker.transfer_progress)} + <%= if chunk_progress = transfer_chunk_progress(@worker.transfer_progress) do %> + Chunk {chunk_progress} + <% end %> {format_throughput(Map.get(@worker.transfer_progress, :bytes_per_second))} ETA {format_eta(Map.get(@worker.transfer_progress, :eta))}
@@ -387,15 +389,12 @@ defmodule ReencodarrWeb.WorkersLive do defp format_transfer_bytes(%{bytes_sent: bytes_sent, total_bytes: total_bytes}), do: format_transfer_bytes(bytes_sent, total_bytes) - defp format_chunk_progress(%{chunk_index: chunk_index, total_chunks: total_chunks}) + defp transfer_chunk_progress(%{chunk_index: chunk_index, total_chunks: total_chunks}) when is_integer(chunk_index) and is_integer(total_chunks) and total_chunks > 0 do "#{chunk_index + 1} / #{total_chunks}" end - defp format_chunk_progress(%{chunk_index: chunk_index}) when is_integer(chunk_index), - do: Integer.to_string(chunk_index) - - defp format_chunk_progress(_), do: "-" + defp transfer_chunk_progress(_progress), do: nil defp status_badge_class(worker) do case worker_phase(worker) do diff --git a/test/reencodarr_web/channels/worker_channel_test.exs b/test/reencodarr_web/channels/worker_channel_test.exs index dd398ca9..1eedadeb 100644 --- a/test/reencodarr_web/channels/worker_channel_test.exs +++ b/test/reencodarr_web/channels/worker_channel_test.exs @@ -826,7 +826,7 @@ defmodule ReencodarrWeb.WorkerChannelTest do session = WorkerSessions.get(socket.assigns.worker_id) assert session.active_video_id == assigned_video_id - assert session.phase == :receiving_input + assert session.phase == :input_ready assert session.transfer_progress.video_id == assigned_video_id assert session.transfer_progress.bytes_sent == content_size assert session.transfer_progress.total_bytes == content_size @@ -844,7 +844,7 @@ defmodule ReencodarrWeb.WorkerChannelTest do session = WorkerSessions.get(socket.assigns.worker_id) assert session.active_video_id == assigned_video_id - assert session.phase == :receiving_input + assert session.phase == :input_ready assert session.transfer_progress.video_id == assigned_video_id end) after diff --git a/test/reencodarr_web/live/workers_live_test.exs b/test/reencodarr_web/live/workers_live_test.exs index 5f434a70..fc2760a6 100644 --- a/test/reencodarr_web/live/workers_live_test.exs +++ b/test/reencodarr_web/live/workers_live_test.exs @@ -79,6 +79,46 @@ defmodule ReencodarrWeb.WorkersLiveTest do refute html =~ "CRF 28.0 -> 95.4 VMAF" end + test "shows HTTP transfer progress without chunk text", %{conn: conn} do + {:ok, _session} = + WorkerSessions.register(%{ + server_worker_id: "worker-server-http", + client_worker_id: "worker-client-http", + protocol_version: 1, + version: "0.10.0", + capabilities: %{"crf_search" => true} + }) + + {:ok, video} = Fixtures.video_fixture(%{state: :analyzed}) + + assert {:ok, _session} = + WorkerSessions.assign_video("worker-server-http", video.id, :receiving_input) + + assert {:ok, _session} = + WorkerSessions.set_transfer_progress("worker-server-http", %{ + job_id: "job-http", + transfer_id: "job-http", + filename: Path.basename(video.path), + percent: 50.0, + bytes_sent: 117_440_512, + total_bytes: 234_881_024, + bytes_per_second: 122_683_392, + eta: 1, + chunk_index: 0, + total_chunks: 0 + }) + + {:ok, view, _html} = live(conn, ~p"/workers") + send(view.pid, {:worker_sessions_updated, %{sessions: WorkerSessions.list()}}) + + html = render(view) + assert html =~ "Receiving Input" + assert html =~ "112.0 MiB / 224.0 MiB" + assert html =~ "117.0 MiB/s" + assert html =~ "ETA 1s" + refute html =~ "Chunk" + end + test "renders the crf search panel when only crf search is active", %{conn: conn} do {:ok, _session} = WorkerSessions.register(%{ From 882c916474e7b95319722e0044457a961388d362 Mon Sep 17 00:00:00 2001 From: Mika Cohen Date: Thu, 9 Jul 2026 11:47:28 -0600 Subject: [PATCH 70/70] Centralize worker transfer state transitions --- .../ab_av1/worker_job_state_machine.ex | 22 +++++++-- lib/reencodarr/ab_av1/worker_sessions.ex | 10 ++-- lib/reencodarr_web/channels/worker_channel.ex | 25 +++------- .../ab_av1/worker_job_state_machine_test.exs | 15 ++++-- .../ab_av1/worker_sessions_test.exs | 4 +- .../channels/worker_channel_test.exs | 46 ++++++++++++++++++- .../reencodarr_web/live/workers_live_test.exs | 46 +++++++++++++++++++ 7 files changed, 134 insertions(+), 34 deletions(-) diff --git a/lib/reencodarr/ab_av1/worker_job_state_machine.ex b/lib/reencodarr/ab_av1/worker_job_state_machine.ex index 6dde7075..ecdf3721 100644 --- a/lib/reencodarr/ab_av1/worker_job_state_machine.ex +++ b/lib/reencodarr/ab_av1/worker_job_state_machine.ex @@ -13,7 +13,7 @@ defmodule Reencodarr.AbAv1.WorkerJobStateMachine do idle: [:receiving_input, :crf_searching], receiving_input: [:receiving_input, :input_ready, :crf_searching, :idle], input_ready: [:receiving_input, :crf_searching, :idle], - crf_searching: [:crf_searching, :receiving_input, :idle] + crf_searching: [:crf_searching, :receiving_input, :input_ready, :idle] } @spec valid_phases() :: [phase()] @@ -62,10 +62,15 @@ defmodule Reencodarr.AbAv1.WorkerJobStateMachine do end @spec set_transfer_progress(map(), map()) :: {:ok, map()} | {:error, :invalid_worker_phase} - def set_transfer_progress(%{phase: phase} = session, progress) - when phase in [:idle, :receiving_input, :input_ready] do + def set_transfer_progress(session, progress), do: record_transfer_progress(session, progress) + + @spec record_transfer_progress(map(), map()) :: {:ok, map()} | {:error, :invalid_worker_phase} + def record_transfer_progress(%{phase: phase} = session, progress) + when phase in [:idle, :receiving_input, :input_ready, :crf_searching] do with {:ok, video_id} <- progress_video_id(session, progress) do - transition(session, :receiving_input, %{ + phase = if transfer_complete?(progress), do: :input_ready, else: :receiving_input + + transition(session, phase, %{ active_video_id: video_id, transfer_progress: progress, crf_search_progress: nil @@ -73,7 +78,7 @@ defmodule Reencodarr.AbAv1.WorkerJobStateMachine do end end - def set_transfer_progress(_session, _progress), do: {:error, :invalid_worker_phase} + def record_transfer_progress(_session, _progress), do: {:error, :invalid_worker_phase} @spec finish_transfer(map()) :: {:ok, map()} | {:error, :invalid_worker_phase} def finish_transfer(%{active_video_id: nil} = session), do: clear_video(session) @@ -107,4 +112,11 @@ defmodule Reencodarr.AbAv1.WorkerJobStateMachine do _ -> {:error, :invalid_worker_phase} end end + + defp transfer_complete?(%{bytes_sent: bytes_sent, total_bytes: total_bytes}) + when is_integer(bytes_sent) and is_integer(total_bytes) and total_bytes > 0, + do: bytes_sent >= total_bytes + + defp transfer_complete?(%{percent: percent}) when is_number(percent), do: percent >= 100 + defp transfer_complete?(_progress), do: false end diff --git a/lib/reencodarr/ab_av1/worker_sessions.ex b/lib/reencodarr/ab_av1/worker_sessions.ex index f3a5be03..b50411fe 100644 --- a/lib/reencodarr/ab_av1/worker_sessions.ex +++ b/lib/reencodarr/ab_av1/worker_sessions.ex @@ -54,7 +54,11 @@ defmodule Reencodarr.AbAv1.WorkerSessions do end def set_transfer_progress(server_worker_id, progress) when is_map(progress) do - GenServer.call(__MODULE__, {:set_transfer_progress, server_worker_id, progress}) + record_transfer_progress(server_worker_id, progress) + end + + def record_transfer_progress(server_worker_id, progress) when is_map(progress) do + GenServer.call(__MODULE__, {:record_transfer_progress, server_worker_id, progress}) end def finish_transfer(server_worker_id) do @@ -145,9 +149,9 @@ defmodule Reencodarr.AbAv1.WorkerSessions do end) end - def handle_call({:set_transfer_progress, server_worker_id, progress}, _from, state) do + def handle_call({:record_transfer_progress, server_worker_id, progress}, _from, state) do update_session_reply(server_worker_id, state, fn session -> - with {:ok, session} <- WorkerJobStateMachine.set_transfer_progress(session, progress) do + with {:ok, session} <- WorkerJobStateMachine.record_transfer_progress(session, progress) do {:ok, %{session | last_seen_at: now()}} end end) diff --git a/lib/reencodarr_web/channels/worker_channel.ex b/lib/reencodarr_web/channels/worker_channel.ex index d61c7648..e34b6880 100644 --- a/lib/reencodarr_web/channels/worker_channel.ex +++ b/lib/reencodarr_web/channels/worker_channel.ex @@ -289,8 +289,7 @@ defmodule ReencodarrWeb.WorkerChannel do :ok <- ensure_active_video(socket, progress.video_id) do {progress, socket} = fill_transfer_rate(socket, progress) - _ = WorkerSessions.set_transfer_progress(worker_id, progress) - _ = maybe_finish_transfer(worker_id, progress) + _ = WorkerSessions.record_transfer_progress(worker_id, progress) Events.broadcast_event(:transfer_progress, Map.put(progress, :worker_id, worker_id)) socket = maybe_send_next_transfer_chunk(socket, progress) @@ -390,7 +389,9 @@ defmodule ReencodarrWeb.WorkerChannel do case WorkerSessions.assign_video(worker_id, video.id, :receiving_input) do {:ok, _session} -> - socket = prepare_transfer(socket, video, target_vmaf, stream?: true) + socket = + prepare_transfer(socket, video, target_vmaf, stream?: true, fail_if_missing?: true) + {:reply, {:ok, WorkerProtocol.work_assigned(video, target_vmaf)}, socket} {:error, reason} -> @@ -431,7 +432,8 @@ defmodule ReencodarrWeb.WorkerChannel do |> assign(:transfer_last_progress_at, nil) |> assign(:transfer_last_progress_bytes, nil) - if Keyword.get(opts, :stream?, true) and File.exists?(video.path) do + if Keyword.get(opts, :stream?, true) and + (File.exists?(video.path) or Keyword.get(opts, :fail_if_missing?, false)) do send(self(), :stream_transfer_chunk) end @@ -890,7 +892,7 @@ defmodule ReencodarrWeb.WorkerChannel do {:noreply, handle_transfer_failure(socket, video_id, :enoent)} else _ = - WorkerSessions.set_transfer_progress( + WorkerSessions.record_transfer_progress( socket.assigns.worker_id, initial_transfer_progress(socket, video) ) @@ -1120,19 +1122,6 @@ defmodule ReencodarrWeb.WorkerChannel do defp maybe_send_next_transfer_chunk(socket, _progress), do: socket - defp maybe_finish_transfer(worker_id, %{bytes_sent: bytes_sent, total_bytes: total_bytes}) - when is_integer(bytes_sent) and is_integer(total_bytes) and total_bytes > 0 and - bytes_sent >= total_bytes do - WorkerSessions.finish_transfer(worker_id) - end - - defp maybe_finish_transfer(worker_id, %{percent: percent}) - when is_number(percent) and percent >= 100 do - WorkerSessions.finish_transfer(worker_id) - end - - defp maybe_finish_transfer(_worker_id, _progress), do: :ok - defp format_file_error(reason) do reason |> :file.format_error() |> List.to_string() end diff --git a/test/reencodarr/ab_av1/worker_job_state_machine_test.exs b/test/reencodarr/ab_av1/worker_job_state_machine_test.exs index 17a07ff1..3fdddd54 100644 --- a/test/reencodarr/ab_av1/worker_job_state_machine_test.exs +++ b/test/reencodarr/ab_av1/worker_job_state_machine_test.exs @@ -3,7 +3,7 @@ defmodule Reencodarr.AbAv1.WorkerJobStateMachineTest do alias Reencodarr.AbAv1.WorkerJobStateMachine - test "moves a worker through input transfer into CRF search without dropping the active video" do + test "complete transfer progress moves the worker to input ready" do session = session() assert {:ok, session} = WorkerJobStateMachine.assign_video(session, 123, :receiving_input) @@ -16,7 +16,7 @@ defmodule Reencodarr.AbAv1.WorkerJobStateMachineTest do percent: 100.0 }) - assert session.phase == :receiving_input + assert session.phase == :input_ready assert session.transfer_progress.percent == 100.0 assert {:ok, session} = WorkerJobStateMachine.finish_transfer(session) @@ -68,7 +68,7 @@ defmodule Reencodarr.AbAv1.WorkerJobStateMachineTest do }) end - test "stale transfer progress cannot move a CRF search back to receiving input" do + test "transfer progress for the active video moves CRF search back to receiving input" do session = session() |> Map.merge(%{ @@ -77,11 +77,16 @@ defmodule Reencodarr.AbAv1.WorkerJobStateMachineTest do crf_search_progress: %{video_id: 123, percent: 25.0} }) - assert {:error, :invalid_worker_phase} = + assert {:ok, session} = WorkerJobStateMachine.set_transfer_progress(session, %{ video_id: 123, - percent: 100.0 + percent: 25.0 }) + + assert session.phase == :receiving_input + assert session.active_video_id == 123 + assert session.transfer_progress.percent == 25.0 + assert is_nil(session.crf_search_progress) end test "clearing active work is the only transition that makes the worker idle" do diff --git a/test/reencodarr/ab_av1/worker_sessions_test.exs b/test/reencodarr/ab_av1/worker_sessions_test.exs index 24176f0b..71404242 100644 --- a/test/reencodarr/ab_av1/worker_sessions_test.exs +++ b/test/reencodarr/ab_av1/worker_sessions_test.exs @@ -120,7 +120,7 @@ defmodule Reencodarr.AbAv1.WorkerSessionsTest do assert listed_session.phase == :crf_searching end - test "moves from receiving input to CRF search without clearing the active video" do + test "complete transfer progress moves to input ready without clearing the active video" do assert {:ok, _session} = WorkerSessions.register(worker_session_attrs()) assert {:ok, session} = WorkerSessions.assign_video("worker-server-1", 123, :receiving_input) @@ -139,7 +139,7 @@ defmodule Reencodarr.AbAv1.WorkerSessionsTest do }) assert session.active_video_id == 123 - assert session.phase == :receiving_input + assert session.phase == :input_ready assert session.transfer_progress.percent == 100.0 assert {:ok, session} = WorkerSessions.finish_transfer("worker-server-1") diff --git a/test/reencodarr_web/channels/worker_channel_test.exs b/test/reencodarr_web/channels/worker_channel_test.exs index 1eedadeb..c675a40c 100644 --- a/test/reencodarr_web/channels/worker_channel_test.exs +++ b/test/reencodarr_web/channels/worker_channel_test.exs @@ -441,7 +441,7 @@ defmodule ReencodarrWeb.WorkerChannelTest do :ok, %{accepted: true, event: "crf_search_progress"} - assert {:error, :invalid_worker_phase} = + assert {:ok, session} = WorkerSessions.set_transfer_progress(server_worker_id, %{ job_id: Integer.to_string(video_id), video_id: video_id, @@ -452,6 +452,10 @@ defmodule ReencodarrWeb.WorkerChannelTest do total_bytes: 8 }) + assert session.phase == :input_ready + assert session.active_video_id == video_id + assert is_nil(session.crf_search_progress) + assert_reply push(socket, "pull_work", %{}), :ok, %{ @@ -661,6 +665,46 @@ defmodule ReencodarrWeb.WorkerChannelTest do Application.delete_env(:reencodarr, :worker_token) end + test "reports transfer failure instead of silently waiting when resend input is unavailable" do + token = "test-worker-token" + Application.put_env(:reencodarr, :worker_token, token) + + missing_path = + Path.join(System.tmp_dir!(), "missing-worker-input-#{System.unique_integer()}.mkv") + + {:ok, video} = + Fixtures.video_fixture(%{ + state: :crf_searching, + path: missing_path, + size: 8, + crf_search_worker_id: "worker-a" + }) + + video_id = video.id + + assert {:ok, socket} = connect(WorkerSocket, %{"token" => token}) + assert {:ok, _join_payload, socket} = subscribe_and_join(socket, "workers:crf_search") + + assert_reply push(socket, "announce", announce_payload(worker_id: "worker-a")), + :ok, + %{accepted: true, protocol_version: 1} + + assert_reply push(socket, "pull_work", %{"input_missing" => true}), + :ok, + %{status: "job_assigned", video_id: ^video_id} + + assert_push "transfer_failed", %{ + status: "transfer_failed", + video_id: ^video_id, + transfer_id: transfer_id + } + + assert transfer_id == Integer.to_string(video_id) + refute_push "transfer_started", _, 50 + after + Application.delete_env(:reencodarr, :worker_token) + end + test "reattaches dispatched work when the session state is gone" do token = "test-worker-token" Application.put_env(:reencodarr, :worker_token, token) diff --git a/test/reencodarr_web/live/workers_live_test.exs b/test/reencodarr_web/live/workers_live_test.exs index fc2760a6..2759047a 100644 --- a/test/reencodarr_web/live/workers_live_test.exs +++ b/test/reencodarr_web/live/workers_live_test.exs @@ -119,6 +119,52 @@ defmodule ReencodarrWeb.WorkersLiveTest do refute html =~ "Chunk" end + test "transfer progress wins over stale crf progress in the worker card", %{conn: conn} do + {:ok, _session} = + WorkerSessions.register(%{ + server_worker_id: "worker-server-stale-crf", + client_worker_id: "worker-client-stale-crf", + protocol_version: 1, + version: "0.10.0", + capabilities: %{"crf_search" => true} + }) + + {:ok, video} = Fixtures.video_fixture(%{state: :crf_searching}) + + assert {:ok, _session} = + WorkerSessions.assign_video("worker-server-stale-crf", video.id) + + assert {:ok, _session} = + WorkerSessions.set_crf_search_progress("worker-server-stale-crf", %{ + video_id: video.id, + percent: 62.0, + fps: 12.5, + eta: 90, + crf: 28.0, + sample_num: 3, + total_samples: 8 + }) + + assert {:ok, _session} = + WorkerSessions.set_transfer_progress("worker-server-stale-crf", %{ + job_id: Integer.to_string(video.id), + transfer_id: Integer.to_string(video.id), + video_id: video.id, + filename: Path.basename(video.path), + percent: 10.0, + bytes_sent: 1_048_576, + total_bytes: 10_485_760 + }) + + {:ok, view, _html} = live(conn, ~p"/workers") + send(view.pid, {:worker_sessions_updated, %{sessions: WorkerSessions.list()}}) + + html = render(view) + assert html =~ "Receiving Input" + refute html =~ "CRF Search" + refute html =~ "Sample 3/8 - CRF 28.0" + end + test "renders the crf search panel when only crf search is active", %{conn: conn} do {:ok, _session} = WorkerSessions.register(%{