diff --git a/ml_peg/app/build_app.py b/ml_peg/app/build_app.py index 2402799f7..bb1566aba 100644 --- a/ml_peg/app/build_app.py +++ b/ml_peg/app/build_app.py @@ -36,12 +36,17 @@ build_page_loading_spinner, build_weight_components, ) +from ml_peg.app.utils.build_frameworks import ( + build_framework_page_layout, + build_framework_summary_tables, + build_framework_views, +) from ml_peg.app.utils.onboarding import ( build_onboarding_modal, register_onboarding_callbacks, ) from ml_peg.app.utils.register_callbacks import ( - register_benchmark_to_category_callback, + register_benchmark_to_group_callback, register_filter_loading_callback, register_filter_tables_callback, ) @@ -51,6 +56,7 @@ ) from ml_peg.app.utils.utils import ( build_level_of_theory_warnings, + framework_sort_key, get_framework_config, get_mlip_column_width, load_model_registry_configs, @@ -543,7 +549,9 @@ def build_category( # Register callback for all benchmark tables -> category table # Category summary table columns add "Score" to name for clarity - register_benchmark_to_category_callback(all_tables, category_to_title) + register_benchmark_to_group_callback( + all_tables, category_to_title, table_id_suffix="-summary-table" + ) return category_views, category_tables, category_weights, framework_ids @@ -604,100 +612,12 @@ def build_category_page_layout( ) -def build_framework_views( - category_views: dict[str, dict[str, object]], - framework_ids: set[str], -) -> dict[str, dict[str, object]]: - """ - Build extra framework-focused page metadata for non-default frameworks. - - Parameters - ---------- - category_views - Category metadata including benchmark layout components. - framework_ids - All framework IDs discovered from benchmark apps. - - Returns - ------- - dict[str, dict[str, object]] - Mapping of framework ID to grouped benchmark layouts by category. - """ - framework_views: dict[str, dict[str, object]] = {} - for framework_id in sorted(framework_ids): - if framework_id == "ml_peg": - continue - - category_groups = [] - for category_name, category_view in category_views.items(): - tests = [ - test["layout"] - for test in category_view["tests"] - if framework_id in test["framework_ids"] - ] - if tests: - category_groups.append({"category": category_name, "tests": tests}) - - if category_groups: - framework_views[framework_id] = { - "framework_id": framework_id, - "label": get_framework_config(framework_id)["label"], - "category_groups": category_groups, - } - - return framework_views - - -def build_framework_page_layout(framework_view: dict[str, object]) -> Div: - """ - Build a framework-focused page containing benchmark sections only. - - Parameters - ---------- - framework_view - Framework page metadata with grouped benchmark layouts by category. - - Returns - ------- - Div - Framework page layout. - """ - framework_label = framework_view["label"] - category_groups = framework_view["category_groups"] - - sections = [] - for group in category_groups: - sections.append(H3(group["category"], style={"marginTop": "26px"})) - sections.append(Div(group["tests"], style={"display": "grid", "gap": "24px"})) - - return Div( - [ - H1(f"{framework_label} Benchmarks"), - Div( - ( - "These benchmarks also remain in their category pages. " - "Framework pages omit the category summary table and reuse the " - "same benchmark controls, so weight and threshold edits stay in " - "sync across both views." - ), - style={ - "fontSize": "13px", - "fontStyle": "italic", - "color": "#64748b", - "marginTop": "8px", - "marginBottom": "8px", - }, - ), - *sections, - ] - ) - - def build_summary_table( tables: dict[str, DataTable], table_id: str = "summary-table", description: str | None = None, weights: dict[str, float] | None = None, + header_labels: dict[str, str] | None = None, ) -> DataTable: """ Build summary table from a set of tables. @@ -712,6 +632,9 @@ def build_summary_table( Description of summary table. Default is None. weights Weights for each column. Default is `None`, which sets all weights to 1. + header_labels + Optional mapping of table key to display header text. Column ids are + unchanged, so only the rendered header differs. Default is None. Returns ------- @@ -749,19 +672,27 @@ def build_summary_table( data = calc_table_scores(data, weights=weights) columns_headers = ("MLIP", "Score") + tuple(key + " Score" for key in tables) - if table_id == "summary-table": - display_headers = { - header: ( - header - if header in {"MLIP", "Score"} or not header.endswith(" Score") - else "\n".join([*header.removesuffix(" Score").split(), "Score"]) + + # Map each column id to its visible header text (labels + line wrapping) + display_headers = {} + for column_id in columns_headers: + header = column_id + if ( + header not in {"MLIP", "Score"} + and header.endswith(" Score") + and header_labels + ): + key = header.removesuffix(" Score") + if key in header_labels: + header = header_labels[key] + " Score" + if table_id != "summary-table": + display_headers[column_id] = _format_summary_column_header(header) + elif header in {"MLIP", "Score"} or not header.endswith(" Score"): + display_headers[column_id] = header + else: + display_headers[column_id] = "\n".join( + [*header.removesuffix(" Score").split(), "Score"] ) - for header in columns_headers - } - else: - display_headers = { - header: _format_summary_column_header(header) for header in columns_headers - } columns = [ {"name": display_headers[header], "id": header} for header in columns_headers @@ -902,6 +833,8 @@ def build_nav( summary_table: DataTable, weight_components: Div, all_apps: dict[str, Dash], + combined_framework_table: DataTable | None = None, + framework_weight_components: Div | None = None, ) -> None: """ Build page layouts and sidebar navigation. @@ -920,15 +853,18 @@ def build_nav( Weight sliders, text boxes and reset button. all_apps Dictionary of all test apps. + combined_framework_table + Frameworks summary table shown on the home page, or None when there are + no external frameworks. + framework_weight_components + Weight controls for the combined framework summary table, or None. """ category_paths = { category_name: _category_to_path(category_name) for category_name in sorted(category_views) } - framework_order = sorted( - framework_views, - key=lambda framework_id: framework_views[framework_id]["label"], - ) + # Frameworks first, then papers, alphabetical by label within each + framework_order = sorted(framework_views, key=framework_sort_key) framework_paths = { framework_id: _framework_to_path(framework_id) for framework_id in framework_order @@ -1011,6 +947,27 @@ def build_nav( ] ) + # Computed + weight stores for each per-framework summary table + framework_state_stores = [] + for framework_view in framework_views.values(): + fw_table = framework_view.get("summary_table") + if fw_table is None: + continue + framework_state_stores.extend( + [ + Store( + id=f"{fw_table.id}-computed-store", + storage_type="session", + data=fw_table.data, + ), + Store( + id=f"{fw_table.id}-weight-store", + storage_type="session", + data=_default_weight_store_data(fw_table), + ), + ] + ) + test_state_stores = [] for app in all_apps.values(): test_state_stores.extend(app.stores) @@ -1023,8 +980,17 @@ def build_nav( ), Store(id="cmap-store", storage_type="local", data="viridis_r"), *category_state_stores, + *framework_state_stores, *test_state_stores, ] + if combined_framework_table is not None: + global_state_stores.append( + Store( + id="framework-summary-table-weight-store", + storage_type="session", + data=_default_weight_store_data(combined_framework_table), + ) + ) full_layout = [ # Start-up mask: covers the page and tutorial with a spinner until the @@ -1092,6 +1058,11 @@ def build_nav( id="summary-table-scores-store", storage_type="session", ), + *( + [Store(id="framework-summary-table-scores-store", storage_type="session")] + if combined_framework_table is not None + else [] + ), Div(global_state_stores, style={"display": "none"}), Div( [ @@ -1159,6 +1130,17 @@ def build_nav( storage_type="session", data=summary_table.data, ), + *( + [ + Store( + id="framework-summary-table-computed-store", + storage_type="session", + data=combined_framework_table.data, + ) + ] + if combined_framework_table is not None + else [] + ), Store( id="filter-recompute-done", storage_type="memory", @@ -1348,6 +1330,7 @@ def select_page( if pathname in (None, "", "/", "/summary"): summary_counts = ( f"{len(category_views)} categories · {len(all_apps)} benchmarks" + f" · {len(framework_views)} frameworks" ) return Div( [ @@ -1389,6 +1372,29 @@ def select_page( ], style={"width": "fit-content"}, ), + *( + [ + H1( + "Frameworks Summary", + style={"marginTop": "32px"}, + ), + Div( + [ + build_download_controls( + combined_framework_table.id, row=True + ), + build_loading_summary_table( + combined_framework_table + ), + Br(), + framework_weight_components, + ], + style={"width": "fit-content"}, + ), + ] + if combined_framework_table is not None + else [] + ), build_faqs(), ] ), sidebar_children @@ -1439,6 +1445,36 @@ def build_full_app(full_app: Dash, category: str = "*", test: str = "*") -> None all_layouts, all_tables, all_frameworks ) framework_views = build_framework_views(cat_views, framework_ids) + # Build per-framework summary tables and the combined framework summary + framework_tables, framework_grouping = build_framework_summary_tables( + all_tables, all_frameworks, framework_views + ) + combined_framework_table = None + framework_weight_components = None + if framework_tables: + combined_framework_table = build_summary_table( + dict( + sorted( + framework_tables.items(), + key=lambda item: framework_sort_key(item[0]), + ) + ), + table_id="framework-summary-table", + header_labels={ + fid: str(framework_views[fid]["label"]) for fid in framework_tables + }, + ) + framework_weight_components = build_weight_components( + header="Weights", + table=combined_framework_table, + include_download_controls=False, + column_widths=combined_framework_table.column_widths, + ) + register_benchmark_to_group_callback( + framework_grouping, + {fid: fid for fid in framework_grouping}, + table_id_suffix="-framework-summary-table", + ) # Build overall summary table summary_table = build_summary_table( dict(sorted(cat_tables.items())), weights=cat_weights @@ -1457,5 +1493,7 @@ def build_full_app(full_app: Dash, category: str = "*", test: str = "*") -> None summary_table, weight_components, all_apps, + combined_framework_table, + framework_weight_components, ) register_onboarding_callbacks() diff --git a/ml_peg/app/utils/build_components.py b/ml_peg/app/utils/build_components.py index 35c63c425..24229171d 100644 --- a/ml_peg/app/utils/build_components.py +++ b/ml_peg/app/utils/build_components.py @@ -303,17 +303,36 @@ def build_weight_components( model_configs = getattr(table, "model_configs", None) # Callbacks to update table scores when table weight dicts change - if table.id != "summary-table": + if table.id == "summary-table": + register_summary_table_callbacks( + initial_rows=table.data, + model_levels=model_levels, + metric_levels=metric_levels, + model_configs=model_configs, + prefix="summary-table", + ) + elif table.id == "framework-summary-table": + register_summary_table_callbacks( + initial_rows=table.data, + model_levels=model_levels, + metric_levels=metric_levels, + model_configs=model_configs, + prefix="framework-summary-table", + ) + elif table.id.endswith("-framework-summary-table"): register_category_table_callbacks( table_id=table.id, use_thresholds=use_thresholds, model_levels=model_levels, metric_levels=metric_levels, model_configs=model_configs, + scores_store_id="framework-summary-table-scores-store", + summary_suffix="-framework-summary-table", ) else: - register_summary_table_callbacks( - initial_rows=table.data, + register_category_table_callbacks( + table_id=table.id, + use_thresholds=use_thresholds, model_levels=model_levels, metric_levels=metric_levels, model_configs=model_configs, diff --git a/ml_peg/app/utils/build_frameworks.py b/ml_peg/app/utils/build_frameworks.py new file mode 100644 index 000000000..b586389ea --- /dev/null +++ b/ml_peg/app/utils/build_frameworks.py @@ -0,0 +1,177 @@ +"""Framework page and summary-table builders for the ML-PEG app.""" + +from __future__ import annotations + +from dash.dash_table import DataTable +from dash.html import H1, H3, Br, Div + +from ml_peg.app.utils.build_components import ( + build_download_controls, + build_loading_summary_table, + build_weight_components, +) +from ml_peg.app.utils.utils import get_framework_config + + +def build_framework_views( + category_views: dict[str, dict[str, object]], + framework_ids: set[str], +) -> dict[str, dict[str, object]]: + """ + Build extra framework-focused page metadata for non-default frameworks. + + Parameters + ---------- + category_views + Category metadata including benchmark layout components. + framework_ids + All framework IDs discovered from benchmark apps. + + Returns + ------- + dict[str, dict[str, object]] + Mapping of framework ID to grouped benchmark layouts by category. + """ + framework_views: dict[str, dict[str, object]] = {} + for framework_id in sorted(framework_ids): + if framework_id == "ml_peg": + continue + + category_groups = [] + for category_name, category_view in category_views.items(): + tests = [ + test["layout"] + for test in category_view["tests"] + if framework_id in test["framework_ids"] + ] + if tests: + category_groups.append({"category": category_name, "tests": tests}) + + if category_groups: + framework_views[framework_id] = { + "framework_id": framework_id, + "label": get_framework_config(framework_id)["label"], + "category_groups": category_groups, + } + + return framework_views + + +def build_framework_summary_tables( + all_tables: dict[str, dict[str, DataTable]], + all_frameworks: dict[str, dict[str, str]], + framework_views: dict[str, dict[str, object]], +) -> tuple[dict[str, DataTable], dict[str, dict[str, DataTable]]]: + """ + Build a per-framework summary table for each framework page. + + Parameters + ---------- + all_tables + Benchmark tables grouped by category. + all_frameworks + Framework ids for each benchmark, grouped by category. + framework_views + Framework page metadata (keys are the frameworks to build tables for; + ``ml_peg`` and empty frameworks are already excluded). + + Returns + ------- + tuple[dict[str, DataTable], dict[str, dict[str, DataTable]]] + Per-framework summary tables keyed by framework id, and the + {framework_id: {benchmark_name: benchmark_table}} grouping. + """ + # Local import avoids a circular import (build_app imports this module). + from ml_peg.app.build_app import build_summary_table + + framework_tables: dict[str, DataTable] = {} + framework_grouping: dict[str, dict[str, DataTable]] = {} + + for framework_id in framework_views: + # Gather benchmark tables tagged with this framework (deterministic order) + benchmarks: dict[str, DataTable] = {} + for category in sorted(all_tables): + for test_name in sorted(all_tables[category]): + if framework_id in all_frameworks[category][test_name]: + benchmarks[test_name] = all_tables[category][test_name] + if not benchmarks: + continue + + summary_table = build_summary_table( + benchmarks, + table_id=f"{framework_id}-framework-summary-table", + description=str(framework_views[framework_id]["label"]), + ) + weight_components = build_weight_components( + header="Weights", + table=summary_table, + include_download_controls=False, + column_widths=getattr(summary_table, "column_widths", None), + ) + framework_views[framework_id]["summary_table"] = summary_table + framework_views[framework_id]["weight_components"] = weight_components + framework_tables[framework_id] = summary_table + framework_grouping[framework_id] = benchmarks + + return framework_tables, framework_grouping + + +def build_framework_page_layout(framework_view: dict[str, object]) -> Div: + """ + Build a framework-focused page with its summary table and benchmark sections. + + Parameters + ---------- + framework_view + Framework page metadata with grouped benchmark layouts by category. + + Returns + ------- + Div + Framework page layout. + """ + framework_label = framework_view["label"] + category_groups = framework_view["category_groups"] + summary_table = framework_view.get("summary_table") + weight_components = framework_view.get("weight_components") + + sections = [] + for group in category_groups: + sections.append(H3(group["category"], style={"marginTop": "26px"})) + sections.append(Div(group["tests"], style={"display": "grid", "gap": "24px"})) + + summary_block = [] + if summary_table is not None: + summary_block = [ + Div( + [ + build_download_controls(summary_table.id, row=True), + build_loading_summary_table(summary_table), + Br(), + weight_components, + ], + style={"width": "fit-content"}, + ), + ] + + return Div( + [ + H1(f"{framework_label} Benchmarks"), + Div( + ( + "These benchmarks also appear on their category pages and " + "share the same benchmark controls, so weight and threshold " + "edits stay in sync across both views." + ), + style={ + "fontSize": "13px", + "fontStyle": "italic", + "color": "#64748b", + "marginTop": "8px", + "marginBottom": "8px", + }, + ), + *summary_block, + *sections, + ] + ) diff --git a/ml_peg/app/utils/frameworks.yml b/ml_peg/app/utils/frameworks.yml index 4bb28f12a..edab5dc1c 100644 --- a/ml_peg/app/utils/frameworks.yml +++ b/ml_peg/app/utils/frameworks.yml @@ -1,11 +1,13 @@ ml_peg: label: ML-PEG + type: framework color: "#334155" text_color: "#ffffff" url: "https://github.com/ddmms/ml-peg" mlip_arena: label: MLIP Arena + type: framework color: "#0f766e" text_color: "#ecfeff" url: "https://huggingface.co/spaces/atomind/mlip-arena" @@ -13,6 +15,7 @@ mlip_arena: mlip_audit: label: MLIP Audit + type: framework color: "#1d4ed8" text_color: "#ffffff" url: "https://github.com/instadeepai/mlipaudit" @@ -20,6 +23,7 @@ mlip_audit: mace-multihead: label: Multihead Cross Learning + type: paper color: "#7c3aed" text_color: "#f5f3ff" url: "https://arxiv.org/abs/2510.25380" @@ -28,6 +32,7 @@ mace-multihead: mace-polar-1: label: MACE-POLAR-1 + type: paper color: "#1d4ed8" text_color: "#eff6ff" url: "https://arxiv.org/abs/2602.19411" diff --git a/ml_peg/app/utils/register_callbacks.py b/ml_peg/app/utils/register_callbacks.py index 0ea193ccc..8563a273f 100644 --- a/ml_peg/app/utils/register_callbacks.py +++ b/ml_peg/app/utils/register_callbacks.py @@ -141,6 +141,7 @@ def register_summary_table_callbacks( model_levels: dict[str, str | None] | None = None, metric_levels: dict[str, str | None] | None = None, model_configs: dict[str, Any] | None = None, + prefix: str = "summary-table", ) -> None: """ Register callbacks to update summary table. @@ -156,14 +157,20 @@ def register_summary_table_callbacks( Mapping from metric column name to its level of theory badge text. model_configs Optional metadata/configuration dictionary for each model. + prefix + Selects which top-level summary table these callbacks update, so the same + logic serves both the overall categories summary ("summary-table") and + the frameworks summary ("framework-summary-table"). It is the table's id + and the prefix of its `-scores-store`, `-weight-store`, and + `-computed-store`. Default is "summary-table". """ default_rows = deepcopy(initial_rows) if initial_rows else [] @callback( - Output("summary-table-computed-store", "data", allow_duplicate=True), - Input("summary-table-scores-store", "data"), - Input("summary-table-weight-store", "data"), - State("summary-table-computed-store", "data"), + Output(f"{prefix}-computed-store", "data", allow_duplicate=True), + Input(f"{prefix}-scores-store", "data"), + Input(f"{prefix}-weight-store", "data"), + State(f"{prefix}-computed-store", "data"), prevent_initial_call=True, ) def update_summary_computed_store( @@ -204,11 +211,11 @@ def update_summary_computed_store( return updated_rows @callback( - Output("summary-table", "data", allow_duplicate=True), - Output("summary-table", "style_data_conditional", allow_duplicate=True), - Output("summary-table", "tooltip_data", allow_duplicate=True), + Output(prefix, "data", allow_duplicate=True), + Output(prefix, "style_data_conditional", allow_duplicate=True), + Output(prefix, "tooltip_data", allow_duplicate=True), Input("selected-models-store", "data"), - Input("summary-table-computed-store", "data"), + Input(f"{prefix}-computed-store", "data"), Input("cmap-store", "data"), prevent_initial_call="initial_duplicate", optional=True, @@ -268,6 +275,8 @@ def register_category_table_callbacks( model_levels: dict[str, str | None] | None = None, metric_levels: dict[str, str | None] | None = None, model_configs: dict[str, Any] | None = None, + scores_store_id: str = "summary-table-scores-store", + summary_suffix: str = "-summary-table", ) -> None: """ Register callback to update table scores when stored values change. @@ -286,6 +295,14 @@ def register_category_table_callbacks( Mapping of metric name -> level of theory metadata. model_configs Optional configuration metadata for each model. + scores_store_id + ID of the global scores store this summary table writes into. Default is + "summary-table-scores-store" (the overall summary's inputs). + summary_suffix + Only a table whose id ends with this suffix writes to the scores store + (the summary tables, not the benchmark tables this is also registered + for). The suffix is stripped from the id to derive the score column key. + Default is "-summary-table". """ @callback( @@ -644,9 +661,9 @@ def sync_table_from_computed_store( return filtered_rows, style, tooltip_data @callback( - Output("summary-table-scores-store", "data", allow_duplicate=True), + Output(scores_store_id, "data", allow_duplicate=True), Input(f"{table_id}-computed-store", "data"), - State("summary-table-scores-store", "data"), + State(scores_store_id, "data"), prevent_initial_call="initial_duplicate", ) def update_scores_store( @@ -668,15 +685,15 @@ def update_scores_store( dict[str, dict[str, float]] Updated summary-score mapping. """ - # Only category summary tables should write to the global store - if not table_id.endswith("-summary-table"): + # Only summary tables should write to the global store + if not table_id.endswith(summary_suffix): raise PreventUpdate if not computed_rows: raise PreventUpdate - # Category table IDs are of form "[category]-summary-table" - category_key = table_id.removesuffix("-summary-table") + " Score" + # Summary table IDs are of form "[key]{summary_suffix}" + category_key = table_id.removesuffix(summary_suffix) + " Score" new_scores = { row["MLIP"]: row["Score"] for row in computed_rows if row.get("MLIP") @@ -692,89 +709,97 @@ def update_scores_store( return patch -def register_benchmark_to_category_callback( - all_tables: dict[str, dict[str, DataTable]], category_to_title: dict[str, str] +def register_benchmark_to_group_callback( + all_tables: dict[str, dict[str, DataTable]], + group_to_id: dict[str, str], + table_id_suffix: str = "-summary-table", ) -> None: """ - Propagate a benchmark table's Score into its category summary table column. + Propagate benchmark Scores into each group's summary table columns. Parameters ---------- all_tables - Tables for all tests, grouped by category. - category_to_title - Dictionary mapping category directory names to their display titles/table IDs. + Benchmark tables grouped by group key (category title or framework id). + group_to_id + Mapping of group key to the id-prefix used in its summary table id. + table_id_suffix + Suffix appended to the id-prefix to form the summary table id. Default is + "-summary-table". """ - all_info = {} - for category, tables in all_tables.items(): - all_info[category] = {} + all_info: dict[str, dict[str, dict]] = {} + for group, tables in all_tables.items(): + all_info[group] = {} for test_name, benchmark_table in tables.items(): - all_info[category][test_name] = { + all_info[group][test_name] = { "benchmark_table_id": benchmark_table.id, "benchmark_column": test_name + " Score", "model_name_map": getattr(benchmark_table, "model_name_map", {}), } + # De-duplicate benchmark computed-store Inputs (a benchmark may be in >1 group) + # For frameworks, a benchmark can be tagged with several frameworks, and Dash + # errors if the same Input appears twice in one callback, so list each once. + benchmark_ids: list[str] = [] + seen: set[str] = set() + for _group, group_info in sorted(all_info.items()): + for _test, info in sorted(group_info.items()): + bid = info["benchmark_table_id"] + if bid not in seen: + seen.add(bid) + benchmark_ids.append(bid) + outputs = [] - inputs = [] - for category, category_info in sorted(all_info.items()): - category_table_id = f"{category_to_title[category]}-summary-table" + state_inputs = [] + for group, _group_info in sorted(all_info.items()): + group_table_id = f"{group_to_id[group]}{table_id_suffix}" outputs.append( - Output(f"{category_table_id}-computed-store", "data", allow_duplicate=True) + Output(f"{group_table_id}-computed-store", "data", allow_duplicate=True) ) + state_inputs.append(State(f"{group_table_id}-weight-store", "data")) + state_inputs.append(State(f"{group_table_id}-computed-store", "data")) - inputs.extend( - [ - State(f"{category_table_id}-weight-store", "data"), - State(f"{category_table_id}-computed-store", "data"), - ] - ) - inputs.extend( - [ - Input(f"{table_info['benchmark_table_id']}-computed-store", "data") - for _, table_info in sorted(category_info.items()) - ] - ) + benchmark_inputs = [Input(f"{bid}-computed-store", "data") for bid in benchmark_ids] - @callback(outputs, inputs, prevent_initial_call=True) - def update_category_from_benchmark(*args) -> list[list[dict]]: + @callback(outputs, state_inputs + benchmark_inputs, prevent_initial_call=True) + def update_group_from_benchmark(*args) -> list: """ - Update cached category summary rows from all benchmarks' cached scores. + Update cached group summary rows from all benchmarks' cached scores. Parameters ---------- *args - States and Inputs for all category summary tables and benchmark tables. - Ordered by category. For each category, the weights, computed store, and - benchmark computed stores are listed sequentially. + Per-group (weight-store, computed-store) States followed by the + de-duplicated benchmark computed-store Inputs. Returns ------- - list[list[dict]] - Refreshed cached rows for each category summary table. + list + Refreshed cached rows (or ``no_update``) for each group summary table. """ - # Rebuild inputs for each category - iterator = iter(args) - patched_outputs = [] + n_state = len(state_inputs) + states = list(args[:n_state]) + benchmark_vals = args[n_state:] + bench_by_id = dict(zip(benchmark_ids, benchmark_vals, strict=True)) - for _category, category_info in sorted(all_info.items()): - category_weights = next(iterator) - current_rows = next(iterator) + state_iter = iter(states) + patched_outputs = [] - updated_rows = [] - for row in current_rows: - updated_row = row.copy() - updated_rows.append(updated_row) + for _group, group_info in sorted(all_info.items()): + group_weights = next(state_iter) + current_rows = next(state_iter) + updated_rows = [row.copy() for row in current_rows] updated_by_mlip = {row["MLIP"]: row for row in updated_rows} - benchmark_changed = False - for _test_name, table_info in sorted(category_info.items()): - benchmark_rows = next(iterator) + for _test, info in sorted(group_info.items()): + benchmark_rows = bench_by_id.get(info["benchmark_table_id"]) + if not benchmark_rows: + continue - name_map = table_info["model_name_map"] - benchmark_column = table_info["benchmark_column"] + name_map = info["model_name_map"] + benchmark_column = info["benchmark_column"] for row in benchmark_rows: display_name = row.get("MLIP") @@ -793,8 +818,8 @@ def update_category_from_benchmark(*args) -> list[list[dict]]: patched_outputs.append(no_update) continue - # Recompute overall category scores using existing utility - rescored_rows, _ = update_score_style(updated_rows, category_weights) + # Recompute overall group scores + rescored_rows, _ = update_score_style(updated_rows, group_weights) patch = Patch() score_changed = False diff --git a/ml_peg/app/utils/utils.py b/ml_peg/app/utils/utils.py index c526db085..125b273cb 100644 --- a/ml_peg/app/utils/utils.py +++ b/ml_peg/app/utils/utils.py @@ -189,6 +189,7 @@ class FrameworkEntry(TypedDict): """Style and link metadata for benchmark framework attribution badges.""" label: str + type: str color: str text_color: str url: NotRequired[str] @@ -1161,8 +1162,16 @@ def load_framework_registry() -> dict[str, FrameworkEntry]: "'label', 'color', or 'text_color' values." ) + entry_type = raw_entry.get("type") + entry_type = ( + entry_type.strip() + if isinstance(entry_type, str) and entry_type.strip() + else "framework" + ) + registry_entry: FrameworkEntry = { "label": label, + "type": entry_type, "color": color, "text_color": text_color, } @@ -1214,3 +1223,22 @@ def get_framework_config(framework_id: str) -> FrameworkEntry: f"Unknown framework identifier '{normalized_id}'. " f"Known framework IDs: {known_ids}." ) from exc + + +def framework_sort_key(framework_id: str) -> tuple[int, str]: + """ + Sort key ordering frameworks before papers, alphabetically by label within each. + + Parameters + ---------- + framework_id + Framework identifier from benchmark app metadata. + + Returns + ------- + tuple[int, str] + Group rank (0 for frameworks, 1 for papers) and lowercased label. + """ + config = get_framework_config(framework_id) + is_paper = config.get("type") == "paper" + return (1 if is_paper else 0, config["label"].casefold())