From 597fcbd921bba68e976b4faa258349616f9c13b8 Mon Sep 17 00:00:00 2001 From: Alessandro Pomponio Date: Tue, 28 Jul 2026 11:08:28 +0100 Subject: [PATCH] fix(core): handle null candidate in sqlite json contains simulation Signed-off-by: Alessandro Pomponio --- ado/metastore/sql/statements.py | 15 +++++++++-- tests/metastore/test_sql_utils.py | 41 ++++++++++++++++++++++++++++++- 2 files changed, 53 insertions(+), 3 deletions(-) diff --git a/ado/metastore/sql/statements.py b/ado/metastore/sql/statements.py index ec45b6f61..64a4c93c4 100644 --- a/ado/metastore/sql/statements.py +++ b/ado/metastore/sql/statements.py @@ -51,7 +51,9 @@ def simulate_json_contains_on_sqlite( In our simulated version, we prepare a subquery that can be used in a WHERE statement that filters rows making sure their ID is one that has all the fields - from the candidate document. + from the candidate document. ``null`` candidates need separate handling because + ``json_tree`` only emits rows for fields that exist in the document, so missing + fields would otherwise produce no row and never match. Args: path (str): The path to the JSON field to check. @@ -70,12 +72,21 @@ def simulate_json_contains_on_sqlite( quoted_table_name = _quote_sql_identifier(table_name) quoted_json_column = _quote_sql_identifier(json_column) quoted_id_column = _quote_sql_identifier(id_column) + parsed_candidate = json.loads(candidate) + + if parsed_candidate is None: + return f""" + {quoted_id_column} IN ( + SELECT {quoted_id_column} FROM {quoted_table_name} + WHERE json_extract({quoted_json_column}, '{path}') IS NULL + ) + """ # noqa: S608 - identifiers are quoted to prevent injection # The subqueries produced by check_field_in_sqlite_json_document need to be # INTERSECT-ed to make sure we only retrieve the identifiers that match all # the subqueries. subqueries = check_field_in_sqlite_json_document( - json.loads(candidate), path, id_column=quoted_id_column + parsed_candidate, path, id_column=quoted_id_column ) return ( diff --git a/tests/metastore/test_sql_utils.py b/tests/metastore/test_sql_utils.py index 0c19d98e0..deacf9a40 100644 --- a/tests/metastore/test_sql_utils.py +++ b/tests/metastore/test_sql_utils.py @@ -4,8 +4,12 @@ import pytest import sqlalchemy -from ado.metastore.sql.statements import table_exists_query +from ado.metastore.sql.statements import ( + simulate_json_contains_on_sqlite, + table_exists_query, +) from ado.metastore.sql.utils import check_table_exists +from tests.conftest import requires_sqlite_3_38 def test_table_exists_query_sqlite() -> None: @@ -31,3 +35,38 @@ def test_check_table_exists_sqlite_memory() -> None: conn.execute(sqlalchemy.text("CREATE TABLE missing (id INTEGER)")) assert check_table_exists(engine, "missing") is True + + +@requires_sqlite_3_38 +def test_simulate_json_contains_on_sqlite_matches_null_and_missing_fields() -> None: + engine = sqlalchemy.create_engine("sqlite:///:memory:") + + with engine.begin() as conn: + conn.execute( + sqlalchemy.text("CREATE TABLE resources (identifier TEXT, data TEXT)") + ) + conn.execute( + sqlalchemy.text( + """ + INSERT INTO resources (identifier, data) + VALUES + ('stored-null', '{"config": {"metadata": {"name": null}}}'), + ('missing-field', '{"config": {"metadata": {}}}'), + ('non-null', '{"config": {"metadata": {"name": "ado"}}}') + """ + ) + ) + + where_clause = simulate_json_contains_on_sqlite( + "$.config.metadata.name", "null" + ) + query = sqlalchemy.text( + f""" + SELECT identifier FROM resources + WHERE {where_clause} + ORDER BY identifier + """ # noqa: S608 - test builds a SQL fragment from a controlled helper + ) + result = conn.execute(query).scalars().all() + + assert result == ["missing-field", "stored-null"]