diff --git a/apps/gust/lib/gust/flows.ex b/apps/gust/lib/gust/flows.ex index 59ec8bc..1a766ba 100644 --- a/apps/gust/lib/gust/flows.ex +++ b/apps/gust/lib/gust/flows.ex @@ -577,6 +577,7 @@ defmodule Gust.Flows do * `:limit` - Required. The maximum number of runs to preload. * `:offset` - Required. The number of runs to skip before starting to preload. * `:status` - Optional. Filters preloaded runs by status when present. + * `:params_search` - Optional. Case-insensitively filters runs by parameter keys or values. ## Returns @@ -589,6 +590,7 @@ defmodule Gust.Flows do limit = Keyword.fetch!(opts, :limit) offset = Keyword.fetch!(opts, :offset) status = Keyword.get(opts, :status) + params_search = Keyword.get(opts, :params_search) runs_q = from r in Run, @@ -596,7 +598,10 @@ defmodule Gust.Flows do limit: ^limit, offset: ^offset - runs_q = maybe_filter_run_status(runs_q, status) + runs_q = + runs_q + |> maybe_filter_run_status(status) + |> maybe_filter_run_params(params_search) Repo.one!( from d in Dag, @@ -611,15 +616,18 @@ defmodule Gust.Flows do ## Parameters * `dag_id` - The identifier of the DAG whose runs should be counted. + * `status` - Optional status to filter by. + * `params_search` - Optional case-insensitive parameter key or value search. ## Returns * The integer count of runs associated with the specified DAG. """ - def count_runs_on_dag(dag_id, status \\ nil) do + def count_runs_on_dag(dag_id, status \\ nil, params_search \\ nil) do Run |> where([r], r.dag_id == ^dag_id) |> maybe_filter_run_status(status) + |> maybe_filter_run_params(params_search) |> Repo.aggregate(:count) end @@ -640,6 +648,22 @@ defmodule Gust.Flows do where(query, [r], r.status == ^status) end + defp maybe_filter_run_params(query, nil), do: query + + defp maybe_filter_run_params(query, params_search) do + case String.trim(params_search) do + "" -> + query + + params_search -> + where( + query, + [r], + fragment("strpos(lower(CAST(? AS text)), lower(?)) > 0", r.params, ^params_search) + ) + end + end + @doc """ Lists all secrets. """ diff --git a/apps/gust/test/flows_test.exs b/apps/gust/test/flows_test.exs index 168fe62..a78351c 100644 --- a/apps/gust/test/flows_test.exs +++ b/apps/gust/test/flows_test.exs @@ -272,6 +272,99 @@ defmodule FlowsTest do assert [failed_run.id] == Enum.map(filtered_dag.runs, & &1.id) end + test "get_dag_by_name_with_runs!/1 searches run parameter keys and values" do + dag = dag_fixture(%{name: "params_filtered"}) + other_dag = dag_fixture(%{name: "other_params_filtered"}) + + key_match = run_fixture(%{dag_id: dag.id, params: %{"CustomerReference" => "unrelated"}}) + value_match = run_fixture(%{dag_id: dag.id, params: %{"reference" => "CUSTOMER-42"}}) + _no_match = run_fixture(%{dag_id: dag.id, params: %{"reference" => "supplier-7"}}) + _other_dag_match = run_fixture(%{dag_id: other_dag.id, params: %{"customer" => "42"}}) + + key_results = + Flows.get_dag_by_name_with_runs!(dag.name, + limit: 30, + offset: 0, + params_search: "customerreference" + ) + + value_results = + Flows.get_dag_by_name_with_runs!(dag.name, + limit: 30, + offset: 0, + params_search: "customer-42" + ) + + assert Enum.map(key_results.runs, & &1.id) == [key_match.id] + assert Enum.map(value_results.runs, & &1.id) == [value_match.id] + end + + test "run parameter search works with status, pagination, and counts" do + dag = dag_fixture(%{name: "paginated_params_filtered"}) + + oldest_match = + run_fixture(%{ + dag_id: dag.id, + status: :failed, + params: %{"customer" => "match"}, + inserted_at: ~N[2021-01-01 00:00:00] + }) + + newest_match = + run_fixture(%{ + dag_id: dag.id, + status: :failed, + params: %{"customer" => "MATCH"}, + inserted_at: ~N[2022-01-01 00:00:00] + }) + + _wrong_status = + run_fixture(%{ + dag_id: dag.id, + status: :succeeded, + params: %{"customer" => "match"} + }) + + _wrong_params = + run_fixture(%{dag_id: dag.id, status: :failed, params: %{"customer" => "other"}}) + + first_page = + Flows.get_dag_by_name_with_runs!(dag.name, + limit: 1, + offset: 0, + status: :failed, + params_search: "match" + ) + + second_page = + Flows.get_dag_by_name_with_runs!(dag.name, + limit: 1, + offset: 1, + status: :failed, + params_search: "match" + ) + + assert Enum.map(first_page.runs, & &1.id) == [newest_match.id] + assert Enum.map(second_page.runs, & &1.id) == [oldest_match.id] + assert Flows.count_runs_on_dag(dag.id, :failed, "match") == 2 + assert Flows.count_runs_on_dag(dag.id, :failed, "missing") == 0 + end + + test "blank run parameter search does not filter runs" do + dag = dag_fixture(%{name: "blank_params_filtered"}) + run = run_fixture(%{dag_id: dag.id, params: %{"customer" => "match"}}) + + loaded_dag = + Flows.get_dag_by_name_with_runs!(dag.name, + limit: 30, + offset: 0, + params_search: " " + ) + + assert Enum.map(loaded_dag.runs, & &1.id) == [run.id] + assert Flows.count_runs_on_dag(dag.id, nil, " ") == 1 + end + test "get_dag_by_name_with_runs!/1 raises when required pagination options are missing" do dag = dag_fixture(%{name: "missing_required_opts"}) diff --git a/apps/gust_web/assets/css/app.css b/apps/gust_web/assets/css/app.css index 61ea6b2..eaa45b6 100644 --- a/apps/gust_web/assets/css/app.css +++ b/apps/gust_web/assets/css/app.css @@ -411,6 +411,14 @@ .app-footer__value { @apply border border-slate-200 bg-white/80 px-2 py-1 text-[11px] font-medium text-slate-700 shadow-sm; } + + .run-params__search-form { + @apply flex h-8 min-w-0 flex-1 items-center gap-2 [&_.fieldset]:mb-0 [&_.fieldset]:h-8 [&_.fieldset]:min-w-0 [&_.fieldset]:flex-1; + } + + .run-status__filter-form { + @apply flex h-8 shrink-0 items-center [&_.fieldset]:mb-0 [&_.fieldset]:h-8; + } } @layer components { diff --git a/apps/gust_web/lib/gust_web/live/run_live/index.ex b/apps/gust_web/lib/gust_web/live/run_live/index.ex index b7ad869..1dd24ee 100644 --- a/apps/gust_web/lib/gust_web/live/run_live/index.ex +++ b/apps/gust_web/lib/gust_web/live/run_live/index.ex @@ -23,9 +23,10 @@ defmodule GustWeb.RunLive.Index do page = String.to_integer(page) selected_status = run_status(params["status"]) selected_status_param = status_param(selected_status) - dag = get_dag_with_runs!(page, page_size, name, selected_status) + params_search = parse_search_params(params["params_search"]) + dag = get_dag_with_runs!(page, page_size, name, selected_status, params_search) - {runs_count, pages} = count_and_pages(dag.id, selected_status, page_size) + {runs_count, pages} = count_and_pages(dag.id, selected_status, params_search, page_size) subscribe_dag_runs(socket, dag) @@ -37,6 +38,8 @@ defmodule GustWeb.RunLive.Index do |> assign(:runs_count, runs_count) |> assign(:page, page) |> assign(:selected_status, selected_status_param) + |> assign(:params_search, params_search) + |> assign(:params_search_form, to_form(%{"params_search" => params_search})) |> assign(:selected_run_ids, []) |> assign(:run_status_options, run_status_options()) |> assign(:all_selected?, false) @@ -44,8 +47,8 @@ defmodule GustWeb.RunLive.Index do |> stream(:runs, dag.runs, reset: true)} end - defp count_and_pages(dag_id, status, size) do - count = Flows.count_runs_on_dag(dag_id, status) + defp count_and_pages(dag_id, status, params_search, size) do + count = Flows.count_runs_on_dag(dag_id, status, params_search) pages = max(div(count + size - 1, size), 1) {count, pages} end @@ -55,20 +58,54 @@ defmodule GustWeb.RunLive.Index do dag_name = socket.assigns.dag_name page_size = socket.assigns.page_size selected_status = socket.assigns.selected_status + params_search = socket.assigns.params_search {:noreply, socket - |> push_patch(to: runs_path(dag_name, page_size, num, selected_status))} + |> push_patch(to: runs_path(dag_name, page_size, num, selected_status, params_search))} end @impl true def handle_event("filter_status", %{"status" => status}, socket) do dag_name = socket.assigns.dag_name page_size = socket.assigns.page_size + params_search = socket.assigns.params_search {:noreply, socket - |> push_patch(to: runs_path(dag_name, page_size, 1, status))} + |> push_patch(to: runs_path(dag_name, page_size, 1, status, params_search))} + end + + @impl true + def handle_event("filter_params", %{"params_search" => params_search}, socket) do + params_search = parse_search_params(params_search) + + {:noreply, + push_patch(socket, + to: + runs_path( + socket.assigns.dag_name, + socket.assigns.page_size, + 1, + socket.assigns.selected_status, + params_search + ) + )} + end + + @impl true + def handle_event("clear_params_search", _params, socket) do + {:noreply, + push_patch(socket, + to: + runs_path( + socket.assigns.dag_name, + socket.assigns.page_size, + 1, + socket.assigns.selected_status, + "" + ) + )} end @impl true @@ -88,7 +125,8 @@ defmodule GustWeb.RunLive.Index do socket.assigns.page, socket.assigns.page_size, socket.assigns.dag_name, - run_status(socket.assigns.selected_status) + run_status(socket.assigns.selected_status), + socket.assigns.params_search ) run_ids = if selected?, do: Enum.map(dag.runs, & &1.id), else: [] @@ -150,7 +188,7 @@ defmodule GustWeb.RunLive.Index do run = Flows.get_run!(run_id) PubSub.subscribe_run(run_id) - if status_matches?(run, socket.assigns.selected_status) do + if run_matches_filters?(run, socket) do {:noreply, socket |> stream_insert(:runs, run, at: 0)} else {:noreply, socket} @@ -164,7 +202,7 @@ defmodule GustWeb.RunLive.Index do ) do run = Flows.get_run!(run_id) - if status_matches?(run, socket.assigns.selected_status) do + if run_matches_filters?(run, socket) do {:noreply, stream_insert(socket, :runs, run)} else {:noreply, @@ -175,10 +213,15 @@ defmodule GustWeb.RunLive.Index do end end - defp get_dag_with_runs!(page, size, name, status) do + defp get_dag_with_runs!(page, size, name, status, params_search) do offset = (page - 1) * size - Flows.get_dag_by_name_with_runs!(name, limit: size, offset: offset, status: status) + Flows.get_dag_by_name_with_runs!(name, + limit: size, + offset: offset, + status: status, + params_search: params_search + ) end defp pretty_json!(value) do @@ -246,11 +289,29 @@ defmodule GustWeb.RunLive.Index do defp status_matches?(_run, ""), do: true defp status_matches?(run, status), do: to_string(run.status) == status + defp params_match?(_run, ""), do: true + + defp params_match?(run, params_search) do + run.params + |> Jason.encode!() + |> String.downcase() + |> String.contains?(String.downcase(params_search)) + end + + defp run_matches_filters?(run, socket) do + status_matches?(run, socket.assigns.selected_status) and + params_match?(run, socket.assigns.params_search) + end + + defp parse_search_params(nil), do: "" + defp parse_search_params(params_search), do: String.trim(params_search) + defp refresh_runs_count(socket) do {runs_count, pages} = count_and_pages( socket.assigns.dag_id, run_status(socket.assigns.selected_status), + socket.assigns.params_search, socket.assigns.page_size ) @@ -265,7 +326,8 @@ defmodule GustWeb.RunLive.Index do socket.assigns.page, socket.assigns.page_size, socket.assigns.dag_name, - run_status(socket.assigns.selected_status) + run_status(socket.assigns.selected_status), + socket.assigns.params_search ) subscribe_dag_runs(socket, dag) @@ -296,9 +358,44 @@ defmodule GustWeb.RunLive.Index do defp maybe_clear_selection(socket, false), do: socket - defp runs_path(name, page_size, page, ""), - do: ~g"/dags/#{name}/runs?page_size=#{page_size}&page=#{page}" + defp runs_path(name, page_size, page, status, params_search) do + query_params = + [{"page_size", page_size}, {"page", page}] + |> maybe_add_query_param("status", status) + |> maybe_add_query_param("params_search", params_search) + + ~g"/dags/#{name}/runs?#{URI.encode_query(query_params)}" + end + + defp pagination_items(current_page, pages) do + total_pages = Enum.max(pages) + + cond do + total_pages <= 7 -> + Enum.to_list(pages) + + current_page <= 4 -> + [1, 2, 3, 4, 5, :ellipsis, total_pages] + + current_page >= total_pages - 3 -> + [1, :ellipsis | Enum.to_list((total_pages - 4)..total_pages)] + + true -> + [ + 1, + :ellipsis, + current_page - 1, + current_page, + current_page + 1, + :ellipsis, + total_pages + ] + end + end + + defp maybe_add_query_param(query_params, _key, ""), do: query_params - defp runs_path(name, page_size, page, status), - do: ~g"/dags/#{name}/runs?page_size=#{page_size}&page=#{page}&status=#{status}" + defp maybe_add_query_param(query_params, key, value) do + query_params ++ [{key, value}] + end end diff --git a/apps/gust_web/lib/gust_web/live/run_live/index.html.heex b/apps/gust_web/lib/gust_web/live/run_live/index.html.heex index f026c89..bdffa24 100644 --- a/apps/gust_web/lib/gust_web/live/run_live/index.html.heex +++ b/apps/gust_web/lib/gust_web/live/run_live/index.html.heex @@ -19,101 +19,132 @@

<.icon name="hero-queue-list" class="h-5 w-5 text-sky-600" /> Runs for {@dag_name}: - {@runs_count} + {@runs_count}

-
+
<.form for={%{}} id="run-batch-form" phx-change="select_runs"> <.table id="runs" rows={@streams.runs}> <:col :let={{_id, run}} label="Select"> @@ -178,6 +209,63 @@
+ +
diff --git a/apps/gust_web/test/gust_web/live/run_live_test.exs b/apps/gust_web/test/gust_web/live/run_live_test.exs index da118f0..48f420d 100644 --- a/apps/gust_web/test/gust_web/live/run_live_test.exs +++ b/apps/gust_web/test/gust_web/live/run_live_test.exs @@ -1,5 +1,5 @@ defmodule GustWeb.RunLiveTest do - alias Gust.Flows + alias Gust.{Flows, Repo} use GustWeb.ConnCase import Phoenix.LiveViewTest @@ -53,6 +53,167 @@ defmodule GustWeb.RunLiveTest do assert html =~ "my_value" end + test "searches run parameter keys and values and stores the search in the URL", %{ + conn: conn, + dag: dag, + run: run_without_match + } do + key_match = run_fixture(%{dag_id: dag.id, params: %{"CustomerReference" => "unrelated"}}) + value_match = run_fixture(%{dag_id: dag.id, params: %{"reference" => "CUSTOMER-42"}}) + + {:ok, index_live, _html} = + live(conn, ~g"/dags/#{dag.name}/runs?page_size=30&page=1") + + assert has_element?(index_live, "#run-search-controls > #run-params-search-form") + assert has_element?(index_live, "#run-search-controls > #status-filter") + + assert has_element?( + index_live, + "#run-params-search[placeholder='Search run params, any key or value']" + ) + + refute has_element?(index_live, "label[for='run-params-search']") + + index_live + |> element("#run-params-search-form") + |> render_submit(%{"params_search" => " customer "}) + + assert_patch( + index_live, + ~g"/dags/#{dag.name}/runs?page_size=30&page=1¶ms_search=customer" + ) + + assert has_element?(index_live, "#run-params-search[value='customer']") + assert has_element?(index_live, "#clear-run-params-search") + assert has_element?(index_live, "#runs-count", "2") + assert has_element?(index_live, "#runs-#{key_match.id}") + assert has_element?(index_live, "#runs-#{value_match.id}") + refute has_element?(index_live, "#runs-#{run_without_match.id}") + end + + test "shows no runs when parameter search has no matches", %{ + conn: conn, + dag: dag, + run: run + } do + {:ok, index_live, _html} = + live( + conn, + ~g"/dags/#{dag.name}/runs?page_size=30&page=1¶ms_search=missing" + ) + + assert has_element?(index_live, "#run-params-search[value='missing']") + assert has_element?(index_live, "#runs-count", "0") + refute has_element?(index_live, "#runs-#{run.id}") + end + + test "clearing parameter search restores runs and preserves the status filter", %{ + conn: conn, + dag: dag + } do + matching_run = + run_fixture(%{dag_id: dag.id, status: :failed, params: %{"customer" => "match"}}) + + other_failed_run = + run_fixture(%{dag_id: dag.id, status: :failed, params: %{"customer" => "other"}}) + + succeeded_run = + run_fixture(%{dag_id: dag.id, status: :succeeded, params: %{"customer" => "match"}}) + + {:ok, index_live, _html} = + live( + conn, + ~g"/dags/#{dag.name}/runs?page_size=30&page=1&status=failed¶ms_search=match" + ) + + assert has_element?(index_live, "#runs-#{matching_run.id}") + refute has_element?(index_live, "#runs-#{other_failed_run.id}") + + index_live + |> element("#clear-run-params-search") + |> render_click() + + assert_patch( + index_live, + ~g"/dags/#{dag.name}/runs?page_size=30&page=1&status=failed" + ) + + assert has_element?(index_live, "#run-params-search[value='']") + refute has_element?(index_live, "#clear-run-params-search") + assert has_element?(index_live, "#runs-#{matching_run.id}") + assert has_element?(index_live, "#runs-#{other_failed_run.id}") + refute has_element?(index_live, "#runs-#{succeeded_run.id}") + end + + test "parameter search works with the status filter", %{conn: conn, dag: dag} do + failed_match = + run_fixture(%{dag_id: dag.id, status: :failed, params: %{"customer" => "match"}}) + + succeeded_match = + run_fixture(%{dag_id: dag.id, status: :succeeded, params: %{"customer" => "match"}}) + + failed_without_match = + run_fixture(%{dag_id: dag.id, status: :failed, params: %{"customer" => "other"}}) + + {:ok, index_live, _html} = + live( + conn, + ~g"/dags/#{dag.name}/runs?page_size=30&page=1¶ms_search=match" + ) + + index_live + |> element("#status-filter") + |> render_change(%{"_target" => "status", "status" => "failed"}) + + assert_patch( + index_live, + ~g"/dags/#{dag.name}/runs?page_size=30&page=1&status=failed¶ms_search=match" + ) + + assert has_element?(index_live, "#runs-#{failed_match.id}") + refute has_element?(index_live, "#runs-#{succeeded_match.id}") + refute has_element?(index_live, "#runs-#{failed_without_match.id}") + end + + test "parameter search is preserved across pagination", %{conn: conn, dag: dag} do + now = DateTime.utc_now() |> DateTime.truncate(:second) + + older_match = + run_fixture(%{ + dag_id: dag.id, + params: %{"customer" => "match"}, + inserted_at: DateTime.add(now, -60) + }) + + newer_match = + run_fixture(%{ + dag_id: dag.id, + params: %{"customer" => "match"}, + inserted_at: DateTime.add(now, 60) + }) + + {:ok, index_live, _html} = + live( + conn, + ~g"/dags/#{dag.name}/runs?page_size=1&page=1¶ms_search=match" + ) + + assert has_element?(index_live, "#runs-#{newer_match.id}") + refute has_element?(index_live, "#runs-#{older_match.id}") + + index_live + |> element("#run-page-2") + |> render_click() + + assert_patch( + index_live, + ~g"/dags/#{dag.name}/runs?page_size=1&page=2¶ms_search=match" + ) + + assert has_element?(index_live, "#runs-#{older_match.id}") + refute has_element?(index_live, "#runs-#{newer_match.id}") + end + test "list runs paged", %{conn: conn, dag: dag, run: _first_run} do page_size = 3 @@ -67,16 +228,54 @@ defmodule GustWeb.RunLiveTest do assert index_live |> has_element?("#runs-#{current_page_run.id}") refute index_live |> has_element?("#runs-#{prev_page_run.id}") - assert index_live |> has_element?("#pages option[value='2']:checked") - refute index_live |> has_element?("#pages option[value='3']") + assert index_live |> has_element?("#runs-table-container + #runs-pagination") + assert index_live |> has_element?("#run-page-2.btn-active[aria-current='page']") + refute index_live |> has_element?("#run-page-3") + refute index_live |> has_element?("#previous-page[disabled]") + assert index_live |> has_element?("#next-page[disabled]") index_live - |> element("#page-select") - |> render_change(%{"_target" => "page", "page" => "1"}) + |> element("#run-page-1") + |> render_click() assert_patch index_live, ~g"/dags/#{dag.name}/runs?page_size=3&page=1" end + test "keeps pagination compact with hundreds of pages", %{conn: conn} do + dag = + dag_fixture(%{ + name: "dag_with_many_runs_#{System.unique_integer([:positive])}" + }) + + now = DateTime.utc_now() |> DateTime.truncate(:second) + + runs = + Enum.map(1..500, fn _index -> + %{ + dag_id: dag.id, + status: :created, + params: %{}, + inserted_at: now, + updated_at: now + } + end) + + {500, nil} = Repo.insert_all(Flows.Run, runs) + + {:ok, index_live, _html} = + live(conn, ~g"/dags/#{dag.name}/runs?page_size=1&page=250") + + assert has_element?(index_live, "#run-page-1") + assert has_element?(index_live, "#run-page-249") + assert has_element?(index_live, "#run-page-250.btn-active[aria-current='page']") + assert has_element?(index_live, "#run-page-251") + assert has_element?(index_live, "#run-page-500") + assert has_element?(index_live, "#runs-pagination .pagination-ellipsis") + refute has_element?(index_live, "#run-page-2") + refute has_element?(index_live, "#run-page-248") + refute has_element?(index_live, "#run-page-252") + end + test "filters runs by status", %{conn: conn, dag: dag, run: created_run} do failed_run = run_fixture(%{dag_id: dag.id, status: :failed}) succeeded_run = run_fixture(%{dag_id: dag.id, status: :succeeded}) @@ -151,8 +350,8 @@ defmodule GustWeb.RunLiveTest do refute index_live |> has_element?("#runs-#{older_failed_run.id}") index_live - |> element("#page-select") - |> render_change(%{"_target" => "page", "page" => "2"}) + |> element("#run-page-2") + |> render_click() assert_patch index_live, ~g"/dags/#{dag.name}/runs?page_size=1&page=2&status=failed" end @@ -365,6 +564,23 @@ defmodule GustWeb.RunLiveTest do refute index_live |> has_element?("#runs-#{new_run.id}") end + test "inserts only newly started runs matching the parameter search", %{conn: conn, dag: dag} do + {:ok, index_live, _html} = + live( + conn, + ~g"/dags/#{dag.name}/runs?page_size=30&page=1¶ms_search=customer-42" + ) + + matching_run = run_fixture(%{dag_id: dag.id, params: %{"customer" => "CUSTOMER-42"}}) + nonmatching_run = run_fixture(%{dag_id: dag.id, params: %{"customer" => "customer-7"}}) + + Gust.PubSub.broadcast_run_started(dag.id, matching_run.id) + Gust.PubSub.broadcast_run_started(dag.id, nonmatching_run.id) + + assert has_element?(index_live, "#runs-#{matching_run.id}") + refute has_element?(index_live, "#runs-#{nonmatching_run.id}") + end + test "run is updated", %{conn: conn, dag: dag, run: run} do {:ok, index_live, _html} = live(conn, ~g"/dags/#{dag.name}/runs?page_size=30&page=1")