Added new slim-ci pipeline with state awareness dbt using clone and defer

.github/workflows/dbt-slim-ci.yml

@@ -0,0 +1,78 @@
+name: dbt CI
+
+on:
+  pull_request:
+    branches:
+      - master
+
+jobs:
+  dbt-run:
+    runs-on: ubuntu-latest
+    environment: DBT_MI
+    defaults:
+      run:
+        working-directory: marketing_insights
+
+    env:
+      DBT_PRIVATE_KEY: ${{ secrets.DBT_PRIVATE_KEY }}
+      DBT_PRIVATE_KEY_PATH: /home/runner/.dbt/rsa_key.p8
+      DBT_SNOWFLAKE_ACCOUNT: ${{ secrets.DBT_SNOWFLAKE_ACCOUNT }}
+      DBT_SNOWFLAKE_ROLE: ${{ secrets.DBT_SNOWFLAKE_ROLE }}
+      DBT_SNOWFLAKE_USER: ${{ secrets.DBT_SNOWFLAKE_USER }}
+      DBT_SCHEMA: pr_${{ github.event.pull_request.number }}_ANALYTICS
+
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0   # needed so we can access master too
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.12.13"
+
+      - name: Install dbt
+        run: |
+          python -m pip install --upgrade pip
+          pip install dbt-core==1.11.8 dbt-snowflake==1.11.4
+
+      - name: Set up dbt profile + private key
+        run: |
+          mkdir -p ~/.dbt
+          cp .dbt/profiles.yml ~/.dbt/profiles.yml
+          echo "$DBT_PRIVATE_KEY" > ~/.dbt/rsa_key.p8
+          chmod 600 ~/.dbt/rsa_key.p8
+
+      - name: Install dbt deps
+        run: dbt deps
+
+      - name: Run dbt debug
+        run: dbt debug --target ci      
+
+      - name: Create state manifest from master
+        run: |
+          git checkout master
+          dbt deps
+          dbt parse --target prod
+          mkdir -p state
+          cp target/manifest.json state/manifest.json
+          git checkout -
+
+      # ---- Clone incrementals first (so is_incremental() is true) ----
+      - name: Clone impacted incremental models into PR schema
+        run: |
+          dbt clone \
+            --target ci \
+            --state ./state \
+            --defer \
+            --select "state:modified+,config.materialized:incremental,state:old"
+
+      # ---- Then slim build ----
+      - name: Slim build (modified + downstream)
+        run: |
+          dbt build \
+            --target ci \
+            --state ./state \
+            --defer \
+            --select "state:modified+"

.gitignore

@@ -2,6 +2,7 @@
 target/
 dbt_packages/
 logs/
+state/
 
 #Secrets
 rsa_key.p8

ReadMe

@@ -28,4 +28,10 @@ using logs folder
 -- Deep dive
 using target dir and look into compile and run folders
 
-kill -9 $(lsof -t -i:8080)
+kill -9 $(lsof -t -i:8080)
+
+dbt compile
+copy the target/manifest.json to state/
+dbt run -m <model_name> --defer --state state
+
+dbt clone --state state

marketing_insights/.dbt/profiles.yml

@@ -10,4 +10,14 @@ marketing_insights:
       type: snowflake
       user: "{{ env_var('DBT_SNOWFLAKE_USER') }}"
       warehouse: COMPUTE_WH
+    prod:
+      account: "{{ env_var('DBT_SNOWFLAKE_ACCOUNT') }}"
+      database: MARKETING_INSIGHTS_CI
+      private_key_path: "{{ env_var('DBT_PRIVATE_KEY_PATH') }}"
+      role: "{{ env_var('DBT_SNOWFLAKE_ROLE') }}"
+      schema: ANALYTICS
+      threads: 1
+      type: snowflake
+      user: "{{ env_var('DBT_SNOWFLAKE_USER') }}"
+      warehouse: COMPUTE_WH      
   target: ci