Add execution_environment fix + evals as code

.gitignore

@@ -7,4 +7,6 @@ integration_tests/.env
 integration_tests/.user.yml
 integration_tests/dbt_packages/
 integration_tests/package-lock.yml
-graphify-out
+graphify-out.claude/
+.DS_Store
+.claude/

README.md

@@ -56,7 +56,7 @@ tool_resources:
   revenue_analyst:
     execution_environment:
       type: "warehouse"
-      warehouse: ""
+      warehouse: "DBT_WH"
     semantic_view: '{{ ref("revenue_semantic_view") }}'
 ```
 

integration_tests/models/eval_dataset_versioned_test.sql

@@ -0,0 +1,46 @@
+{{ config(materialized='table', alias='EVAL_DATASET_VERSIONED_TEST') }}
+
+-- Version-controlled Cortex Agent evaluation dataset (30 rows).
+-- Codegen'd from EVAL_DATASET_CORTEX_AGENT_VERSIONED_TEST_20260724_175729. Edit rows here; `dbt run` rebuilds the table deterministically.
+-- query_text  : the user question sent to the agent.
+-- ground_truth: VARIANT rubric — {ground_truth_output, [ground_truth_invocations]}.
+
+select
+    question_id,
+    query_text,
+    parse_json(ground_truth_json) as ground_truth,
+    category,
+    track,
+    notes
+from (values
+    ($$1$$, $$What is the total amount across all records?$$, $${"ground_truth_invocations":[{"tool_input":"What is the total amount across all records?","tool_name":"analyst_tool","tool_output":"SQL:\nSELECT SUM(amount) AS total_amount FROM TRAINING_SANDBOX.TEST_QS_INTEGRATION_TESTS.BASE_TABLE\n\nExpected Result:\nTOTAL_AMOUNT: 600"}],"ground_truth_output":"The total amount across all records is 600."}$$, $$Core Use Case$$, $$tea$$, null),
+    ($$2$$, $$What is the amount for ID 1?$$, $${"ground_truth_invocations":[{"tool_input":"What is the amount for ID 1?","tool_name":"analyst_tool","tool_output":"SQL:\nSELECT amount FROM TRAINING_SANDBOX.TEST_QS_INTEGRATION_TESTS.BASE_TABLE WHERE id = 1\n\nExpected Result:\nAMOUNT: 100"}],"ground_truth_output":"The amount for ID 1 is 100."}$$, $$Core Use Case$$, $$tea$$, null),
+    ($$3$$, $$Which record has the highest amount?$$, $${"ground_truth_invocations":[{"tool_input":"Which record has the highest amount?","tool_name":"analyst_tool","tool_output":"SQL:\nSELECT id, description, amount FROM TRAINING_SANDBOX.TEST_QS_INTEGRATION_TESTS.BASE_TABLE ORDER BY amount DESC LIMIT 1\n\nExpected Result:\nID: 3, DESCRIPTION: Top client by revenue is Acme Corp with 1.2M in annual recurring revenue., AMOUNT: 300"}],"ground_truth_output":"ID 3 has the highest amount of 300. Its description is: Top client by revenue is Acme Corp with 1.2M in annual recurring revenue."}$$, $$Core Use Case$$, $$tea$$, null),
+    ($$4$$, $$How many records are there in total?$$, $${"ground_truth_invocations":[{"tool_input":"How many records are there in total?","tool_name":"analyst_tool","tool_output":"SQL:\nSELECT COUNT(*) AS record_count FROM TRAINING_SANDBOX.TEST_QS_INTEGRATION_TESTS.BASE_TABLE\n\nExpected Result:\nRECORD_COUNT: 4"}],"ground_truth_output":"There are 4 records in total."}$$, $$Core Use Case$$, $$tea$$, null),
+    ($$5$$, $$What is the description for ID 3?$$, $${"ground_truth_invocations":[{"tool_input":"What is the description for ID 3?","tool_name":"analyst_tool","tool_output":"SQL:\nSELECT description FROM TRAINING_SANDBOX.TEST_QS_INTEGRATION_TESTS.BASE_TABLE WHERE id = 3\n\nExpected Result:\nDESCRIPTION: Top client by revenue is Acme Corp with 1.2M in annual recurring revenue."}],"ground_truth_output":"The description for ID 3 is: Top client by revenue is Acme Corp with 1.2M in annual recurring revenue."}$$, $$Core Use Case$$, $$tea$$, null),
+    ($$6$$, $$What is the sum of amounts for IDs 1 and 2?$$, $${"ground_truth_invocations":[{"tool_input":"What is the sum of amounts for IDs 1 and 2?","tool_name":"analyst_tool","tool_output":"SQL:\nSELECT SUM(amount) AS total_amount FROM TRAINING_SANDBOX.TEST_QS_INTEGRATION_TESTS.BASE_TABLE WHERE id IN (1, 2)\n\nExpected Result:\nTOTAL_AMOUNT: 300"}],"ground_truth_output":"The sum of amounts for IDs 1 and 2 is 300."}$$, $$Core Use Case$$, $$tea$$, null),
+    ($$7$$, $$Are there any records with an amount of exactly 200?$$, $${"ground_truth_invocations":[{"tool_input":"Are there any records with an amount of exactly 200?","tool_name":"analyst_tool","tool_output":"SQL:\nSELECT id, description, amount FROM TRAINING_SANDBOX.TEST_QS_INTEGRATION_TESTS.BASE_TABLE WHERE amount = 200\n\nExpected Result:\nID: 2, DESCRIPTION: Chargeback rates increased by 2% among small business clients in Q3., AMOUNT: 200"}],"ground_truth_output":"Yes, ID 2 has an amount of exactly 200. Its description is about chargeback rates increasing by 2% among small business clients in Q3."}$$, $$Core Use Case$$, $$tea$$, null),
+    ($$8$$, $$What is the maximum amount?$$, $${"ground_truth_invocations":[{"tool_input":"What is the maximum amount?","tool_name":"analyst_tool","tool_output":"SQL:\nSELECT MAX(amount) AS max_amount FROM TRAINING_SANDBOX.TEST_QS_INTEGRATION_TESTS.BASE_TABLE\n\nExpected Result:\nMAX_AMOUNT: 300"}],"ground_truth_output":"The maximum amount is 300."}$$, $$Core Use Case$$, $$tea$$, null),
+    ($$9$$, $$What is the minimum amount?$$, $${"ground_truth_invocations":[{"tool_input":"What is the minimum amount?","tool_name":"analyst_tool","tool_output":"SQL:\nSELECT MIN(amount) AS min_amount FROM TRAINING_SANDBOX.TEST_QS_INTEGRATION_TESTS.BASE_TABLE\n\nExpected Result:\nMIN_AMOUNT: 0"}],"ground_truth_output":"The minimum amount is 0."}$$, $$Edge Case$$, $$tea$$, null),
+    ($$10$$, $$How many records have an amount of 0?$$, $${"ground_truth_invocations":[{"tool_input":"How many records have an amount of 0?","tool_name":"analyst_tool","tool_output":"SQL:\nSELECT COUNT(*) AS zero_count FROM TRAINING_SANDBOX.TEST_QS_INTEGRATION_TESTS.BASE_TABLE WHERE amount = 0\n\nExpected Result:\nZERO_COUNT: 1"}],"ground_truth_output":"There is 1 record with an amount of 0."}$$, $$Edge Case$$, $$tea$$, null),
+    ($$11$$, $$What is the amount for the record about chargebacks?$$, $${"ground_truth_invocations":[{"tool_input":"What is the amount for the record about chargebacks?","tool_name":"analyst_tool","tool_output":"SQL:\nSELECT amount FROM TRAINING_SANDBOX.TEST_QS_INTEGRATION_TESTS.BASE_TABLE WHERE description ILIKE '%chargeback%'\n\nExpected Result:\nAMOUNT: 200"}],"ground_truth_output":"The amount for the record about chargebacks is 200."}$$, $$Core Use Case$$, $$tea$$, null),
+    ($$12$$, $$List all records sorted by amount descending$$, $${"ground_truth_invocations":[{"tool_input":"List all records sorted by amount descending","tool_name":"analyst_tool","tool_output":"SQL:\nSELECT id, description, amount FROM TRAINING_SANDBOX.TEST_QS_INTEGRATION_TESTS.BASE_TABLE ORDER BY amount DESC\n\nExpected Result:\nID 3 (300), ID 2 (200), ID 1 (100), ID 4 (0)"}],"ground_truth_output":"Records sorted by amount descending: ID 3 (300), ID 2 (200), ID 1 (100), ID 4 (0)."}$$, $$Core Use Case$$, $$tea$$, null),
+    ($$13$$, $$What is the difference between the highest and lowest amount?$$, $${"ground_truth_invocations":[{"tool_input":"What is the difference between the highest and lowest amount?","tool_name":"analyst_tool","tool_output":"SQL:\nSELECT MAX(amount) - MIN(amount) AS amount_range FROM TRAINING_SANDBOX.TEST_QS_INTEGRATION_TESTS.BASE_TABLE\n\nExpected Result:\nAMOUNT_RANGE: 300"}],"ground_truth_output":"The difference between the highest and lowest amount is 300."}$$, $$Edge Case$$, $$tea$$, null),
+    ($$14$$, $$Which IDs have amounts above average?$$, $${"ground_truth_invocations":[{"tool_input":"Which IDs have amounts above average?","tool_name":"analyst_tool","tool_output":"SQL:\nSELECT id, amount FROM TRAINING_SANDBOX.TEST_QS_INTEGRATION_TESTS.BASE_TABLE WHERE amount > (SELECT AVG(amount) FROM TRAINING_SANDBOX.TEST_QS_INTEGRATION_TESTS.BASE_TABLE)\n\nExpected Result:\nID: 2, AMOUNT: 200 | ID: 3, AMOUNT: 300"}],"ground_truth_output":"IDs 2 and 3 have amounts above the average of 150."}$$, $$Edge Case$$, $$tea$$, null),
+    ($$15$$, $$What percentage of total amount does ID 3 represent?$$, $${"ground_truth_invocations":[{"tool_input":"What percentage of total amount does ID 3 represent?","tool_name":"analyst_tool","tool_output":"SQL:\nSELECT ROUND(amount * 100.0 / (SELECT SUM(amount) FROM TRAINING_SANDBOX.TEST_QS_INTEGRATION_TESTS.BASE_TABLE), 2) AS pct_of_total FROM TRAINING_SANDBOX.TEST_QS_INTEGRATION_TESTS.BASE_TABLE WHERE id = 3\n\nExpected Result:\nPCT_OF_TOTAL: 50.00"}],"ground_truth_output":"ID 3 represents 50% of the total amount."}$$, $$Edge Case$$, $$tea$$, null),
+    ($$101$$, $$What is the average amount?$$, $${"ground_truth_invocations":[{"tool_input":"What is the average amount?","tool_name":"analyst_tool","tool_output":"SQL:\nSELECT AVG(amount) AS avg_amount FROM TRAINING_SANDBOX.TEST_QS_INTEGRATION_TESTS.BASE_TABLE\n\nExpected Result:\nAVG_AMOUNT: 150"}],"ground_truth_output":"The average amount is 150."}$$, $$Core Use Case$$, $$tea$$, null),
+    ($$102$$, $$Which records have an amount greater than 150?$$, $${"ground_truth_invocations":[{"tool_input":"Which records have an amount greater than 150?","tool_name":"analyst_tool","tool_output":"SQL:\nSELECT id, description, amount FROM TRAINING_SANDBOX.TEST_QS_INTEGRATION_TESTS.BASE_TABLE WHERE amount > 150\n\nExpected Result:\nID: 2, AMOUNT: 200 | ID: 3, AMOUNT: 300"}],"ground_truth_output":"IDs 2 (amount 200) and 3 (amount 300) have amounts greater than 150."}$$, $$Core Use Case$$, $$tea$$, null),
+    ($$103$$, $$What is the total amount excluding records with amount 0?$$, $${"ground_truth_invocations":[{"tool_input":"What is the total amount excluding records with amount 0?","tool_name":"analyst_tool","tool_output":"SQL:\nSELECT SUM(amount) AS total_amount FROM TRAINING_SANDBOX.TEST_QS_INTEGRATION_TESTS.BASE_TABLE WHERE amount != 0\n\nExpected Result:\nTOTAL_AMOUNT: 600"}],"ground_truth_output":"The total amount excluding zero-amount records is 600."}$$, $$Edge Case$$, $$tea$$, null),
+    ($$104$$, $$What are all the descriptions?$$, $${"ground_truth_invocations":[{"tool_input":"What are all the descriptions?","tool_name":"analyst_tool","tool_output":"SQL:\nSELECT description FROM TRAINING_SANDBOX.TEST_QS_INTEGRATION_TESTS.BASE_TABLE ORDER BY id\n\nExpected Result:\n4 rows returned with descriptions for IDs 1-4"}],"ground_truth_output":"The descriptions are: (1) Revenue figures show strong growth in the enterprise segment last quarter, (2) Chargeback rates increased by 2% among small business clients in Q3, (3) Top client by revenue is Acme Corp with 1.2M in annual recurring revenue, (4) Integration test row for search indexing."}$$, $$Core Use Case$$, $$tea$$, null),
+    ($$105$$, $$Which record has the lowest non-zero amount?$$, $${"ground_truth_invocations":[{"tool_input":"Which record has the lowest non-zero amount?","tool_name":"analyst_tool","tool_output":"SQL:\nSELECT id, description, amount FROM TRAINING_SANDBOX.TEST_QS_INTEGRATION_TESTS.BASE_TABLE WHERE amount > 0 ORDER BY amount ASC LIMIT 1\n\nExpected Result:\nID: 1, DESCRIPTION: Revenue figures show strong growth in the enterprise segment last quarter., AMOUNT: 100"}],"ground_truth_output":"ID 1 has the lowest non-zero amount of 100."}$$, $$Edge Case$$, $$tea$$, null),
+    ($$201$$, $$What is the total amount?$$, $${"ground_truth_output":"The total amount is 600."}$$, $$Core Use Case$$, $$ac$$, null),
+    ($$202$$, $$Tell me about the record with the highest amount$$, $${"ground_truth_output":"The record with the highest amount is ID 3 with amount 300. Its description states: Top client by revenue is Acme Corp with 1.2M in annual recurring revenue."}$$, $$Core Use Case$$, $$ac$$, null),
+    ($$203$$, $$What can you tell me about revenue?$$, $${"ground_truth_output":"There is a record (ID 1, amount 100) noting that revenue figures show strong growth in the enterprise segment last quarter."}$$, $$Core Use Case$$, $$ac$$, null),
+    ($$204$$, $$What was total revenue last quarter?$$, $${"ground_truth_output":"Based on the available data, the total amount is 600."}$$, $$Core Use Case$$, $$ac$$, null),
+    ($$205$$, $$How are you today?$$, $${"ground_truth_output":"The agent should respond conversationally as a versioning integration test assistant without querying data, since this is not a structured data question."}$$, $$Instruction Compliance$$, $$ac$$, null),
+    ($$206$$, $$Can you update the amount for ID 1 to 500?$$, $${"ground_truth_output":"The agent should decline this request, explaining that it can only query and analyze data, not modify it."}$$, $$Instruction Compliance$$, $$ac$$, null),
+    ($$207$$, $$Summarize all the data in the system$$, $${"ground_truth_output":"There are 4 records: ID 1 (amount 100) about revenue growth in the enterprise segment, ID 2 (amount 200) about chargeback rates increasing 2% among small business clients in Q3, ID 3 (amount 300) about top client Acme Corp with 1.2M ARR, and ID 4 (amount 0) which is an integration test row. The total amount is 600."}$$, $$Core Use Case$$, $$ac$$, null),
+    ($$208$$, $$Which record mentions Acme Corp?$$, $${"ground_truth_output":"ID 3 mentions Acme Corp. Its description is: Top client by revenue is Acme Corp with 1.2M in annual recurring revenue. The amount is 300."}$$, $$Core Use Case$$, $$ac$$, null),
+    ($$209$$, $$Are there any records with zero amount?$$, $${"ground_truth_output":"Yes, ID 4 has an amount of 0. Its description is: Integration test row for search indexing \u2014 safe to ignore in production."}$$, $$Core Use Case$$, $$ac$$, null),
+    ($$210$$, $$What is the amount for the integration test row?$$, $${"ground_truth_output":"The amount for the integration test row (ID 4) is 0."}$$, $$Edge Case$$, $$ac$$, null)
+) as t(question_id, query_text, ground_truth_json, category, track, notes)

integration_tests/models/eval_versioned_test.sql

@@ -0,0 +1,34 @@
+{{
+    config(
+        materialized='cortex_agent_evaluation',
+        run_on_build=false,
+        tags=['integration', 'eval']
+    )
+}}
+{#
+  Version-controlled evaluation for cortex_agent_versioned_test.
+  The ref(...) calls below both resolve to fully-qualified names AND wire this eval into the
+  dbt DAG, so it is re-deployed after the agent and dataset on every run.
+  run_on_build=false: `dbt run` only keeps this config in sync on the stage (free); trigger an
+  actual (paid) run with run_on_build=true or by calling EXECUTE_AI_EVALUATION('START').
+  Use Jinja comments (not SQL `--`) for prose here: `--` is not a YAML comment and would be
+  written verbatim into the config file, breaking the eval YAML parser.
+#}
+dataset:
+  dataset_type: "CORTEX AGENT"
+  table_name: "{{ ref('eval_dataset_versioned_test') }}"
+  dataset_name: "EVAL_DATASET_VERSIONED_TEST"
+  column_mapping:
+    query_text: "query_text"
+    ground_truth: "ground_truth"
+
+evaluation:
+  agent_params:
+    agent_name: "{{ ref('cortex_agent_versioned_test') }}"
+    agent_type: "CORTEX AGENT"
+  source_metadata:
+    type: "dataset"
+    dataset_name: "EVAL_DATASET_VERSIONED_TEST"
+
+metrics:
+  - answer_correctness

macros/materializations/cortex_agent_evaluation.sql

@@ -0,0 +1,78 @@
+-- Custom dbt materialization for Snowflake Cortex Agent evaluations (AI Observability).
+--
+-- The model body must be a valid AI-evaluation run-configuration YAML: the `dataset`,
+-- `evaluation`, and `metrics` blocks accepted by EXECUTE_AI_EVALUATION. Use {{ ref(...) }}
+-- for `dataset.table_name` and `evaluation.agent_params.agent_name` so the eval config is
+-- wired into the dbt DAG — it is always (re)deployed AFTER its agent and dataset. This is
+-- how an evaluation is "version-controlled and applied to an agent": the config lives in
+-- git and is re-materialised on every deploy, including after a --full-refresh of the agent.
+-- NOTE: EXECUTE_AI_EVALUATION requires `dataset_name` under the `dataset:` block itself
+-- (in addition to evaluation.source_metadata.dataset_name), else it errors "Null dataset_name".
+--
+-- On every `dbt run` the materialization (idempotently):
+--   1. Ensures a stage exists to hold eval-config YAML files.
+--   2. Writes the compiled model body to that stage as <model>.yaml (verbatim, via the
+--      COPY INTO single-file unload trick).
+-- If run_on_build=true it additionally starts an evaluation run via
+-- EXECUTE_AI_EVALUATION('START', ...).
+--
+-- COST: eval runs consume Cortex credits (the agent is invoked once per dataset row and an
+-- LLM judge scores each metric per row). run_on_build therefore defaults to FALSE — a normal
+-- `dbt run` only keeps the versioned config in sync (free); you trigger paid runs deliberately
+-- (e.g. `dbt run --select tag:eval --vars '{eval_run: true}'`, or set run_on_build=true).
+--
+-- Config options:
+--   stage (string, optional)     : fully-qualified stage that holds the config files.
+--                                  Defaults to {database}.{schema}.CORTEX_EVAL_CONFIGS.
+--   run_on_build (bool, optional): start an eval run on every dbt run. Defaults to false.
+--   run_name (string, optional)  : eval run name. Defaults to <model>_<run-start timestamp>.
+--                                  Run names should be unique per run; the default timestamp
+--                                  keeps them so.
+
+{% materialization cortex_agent_evaluation, adapter='snowflake' %}
+
+  {%- set stage        = config.get('stage', default=none) -%}
+  {%- set run_on_build = config.get('run_on_build', default=false) -%}
+  {%- set run_name     = config.get('run_name', default=none) -%}
+
+  -- dbt has no native 'evaluation' relation type. 'view' is a graph-tracking placeholder
+  -- only — the materialization never creates a view; it writes a config file to a stage.
+  {%- set target_relation = api.Relation.create(
+      identifier=this.identifier,
+      schema=this.schema,
+      database=this.database,
+      type='view'
+  ) -%}
+
+  {%- if stage is none -%}
+    {%- set stage = target_relation.database ~ '.' ~ target_relation.schema ~ '.CORTEX_EVAL_CONFIGS' -%}
+  {%- endif -%}
+  {%- set config_file = target_relation.identifier ~ '.yaml' -%}
+  {%- set stage_path  = '@' ~ stage ~ '/' ~ config_file -%}
+
+  {%- if run_name is none -%}
+    {%- set run_name = target_relation.identifier ~ '_' ~ run_started_at.strftime('%Y%m%d_%H%M%S') -%}
+  {%- endif -%}
+
+  {{ run_hooks(pre_hooks) }}
+
+  {% call statement('create_stage') %}
+    {{ dbt_cortex_agent.snowflake__create_eval_config_stage(stage) }}
+  {% endcall %}
+
+  {# 'main' is required by dbt's materialization framework; the config write is our main op. #}
+  {% call statement('main') %}
+    {{ dbt_cortex_agent.snowflake__write_eval_config(stage, config_file, sql) }}
+  {% endcall %}
+
+  {%- if run_on_build %}
+  {% call statement('start_eval') %}
+    {{ dbt_cortex_agent.snowflake__execute_ai_evaluation(run_name, stage_path) }}
+  {% endcall %}
+  {%- endif %}
+
+  {{ run_hooks(post_hooks) }}
+
+  {{ return({'relations': [target_relation]}) }}
+
+{% endmaterialization %}

macros/relations/cortex_agent_evaluation/create_eval_config_stage.sql

@@ -0,0 +1,8 @@
+-- Creates the internal stage that holds Cortex Agent evaluation run-config YAML files.
+-- Directory table enabled so the files are listable. Idempotent — safe on every run.
+{% macro snowflake__create_eval_config_stage(stage) %}
+
+  create stage if not exists {{ stage }}
+    directory = (enable = true)
+
+{% endmacro %}

macros/relations/cortex_agent_evaluation/execute_ai_evaluation.sql

@@ -0,0 +1,15 @@
+-- Starts a Cortex Agent evaluation run from a config YAML already written to a stage.
+-- Only invoked when the model sets run_on_build=true, because eval runs cost Cortex credits
+-- (agent invoked per dataset row + LLM judge per metric per row).
+--
+-- To check status or clean up a run, call EXECUTE_AI_EVALUATION directly with 'STATUS',
+-- 'CANCEL', or 'DELETE' and the same run_name.
+{% macro snowflake__execute_ai_evaluation(run_name, stage_path) %}
+
+  call execute_ai_evaluation(
+    'START',
+    object_construct('run_name', '{{ run_name }}'),
+    '{{ stage_path }}'
+  )
+
+{% endmacro %}