From 8af04c68d20777d7e73d08f263602492b70a3b2a Mon Sep 17 00:00:00 2001
From: Kanaka Pothina <techminionacademy@Kirans-Tech-Minion-Academy.local>
Date: Sat, 16 May 2026 20:20:28 -0400
Subject: [PATCH] Added new slim-ci pipeline with state awareness dbt using
 clone and defer

---
 .github/workflows/dbt-ci.yml      | 96 +++++++++++++++----------------
 .github/workflows/dbt-slim-ci.yml | 78 +++++++++++++++++++++++++
 .gitignore                        |  1 +
 ReadMe                            |  8 ++-
 4 files changed, 134 insertions(+), 49 deletions(-)
 create mode 100644 .github/workflows/dbt-slim-ci.yml

diff --git a/.github/workflows/dbt-ci.yml b/.github/workflows/dbt-ci.yml
index 592773c..03cf94b 100644
--- a/.github/workflows/dbt-ci.yml
+++ b/.github/workflows/dbt-ci.yml
@@ -1,48 +1,48 @@
-name: dbt CI
-
-on:
-  pull_request:
-    branches:
-      - master
-
-jobs:
-  dbt-run:
-    runs-on: ubuntu-latest
-    environment: DBT_MI
-    defaults:
-      run:
-        working-directory: marketing_insights
-
-    env:
-      DBT_PRIVATE_KEY: ${{ secrets.DBT_PRIVATE_KEY }}
-      DBT_PRIVATE_KEY_PATH: /home/runner/.dbt/rsa_key.p8
-
-    steps:
-      - name: Checkout repo
-        uses: actions/checkout@v3
-
-      - name: Set up Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: "3.12.13"
-
-      - name: Install dbt
-        run: |
-          python -m pip install --upgrade pip
-          pip install dbt-core==1.11.8 dbt-snowflake==1.11.4
-
-      - name: Set up dbt profile + private key
-        run: |
-          mkdir -p ~/.dbt
-          cp .dbt/profiles.yml ~/.dbt/profiles.yml
-          echo "$DBT_PRIVATE_KEY" > ~/.dbt/rsa_key.p8
-          chmod 600 ~/.dbt/rsa_key.p8
-
-      - name: Install dbt deps
-        run: dbt deps
-
-      - name: Run dbt debug
-        run: dbt debug --target ci
-
-      - name: Run dbt
-        run: dbt build --target ci
\ No newline at end of file
+# name: dbt CI
+
+# on:
+#   pull_request:
+#     branches:
+#       - master
+
+# jobs:
+#   dbt-run:
+#     runs-on: ubuntu-latest
+#     environment: DBT_MI
+#     defaults:
+#       run:
+#         working-directory: marketing_insights
+
+#     env:
+#       DBT_PRIVATE_KEY: ${{ secrets.DBT_PRIVATE_KEY }}
+#       DBT_PRIVATE_KEY_PATH: /home/runner/.dbt/rsa_key.p8
+
+#     steps:
+#       - name: Checkout repo
+#         uses: actions/checkout@v3
+
+#       - name: Set up Python
+#         uses: actions/setup-python@v4
+#         with:
+#           python-version: "3.12.13"
+
+#       - name: Install dbt
+#         run: |
+#           python -m pip install --upgrade pip
+#           pip install dbt-core==1.11.8 dbt-snowflake==1.11.4
+
+#       - name: Set up dbt profile + private key
+#         run: |
+#           mkdir -p ~/.dbt
+#           cp .dbt/profiles.yml ~/.dbt/profiles.yml
+#           echo "$DBT_PRIVATE_KEY" > ~/.dbt/rsa_key.p8
+#           chmod 600 ~/.dbt/rsa_key.p8
+
+#       - name: Install dbt deps
+#         run: dbt deps
+
+#       - name: Run dbt debug
+#         run: dbt debug --target ci
+
+#       - name: Run dbt
+#         run: dbt build --target ci
\ No newline at end of file
diff --git a/.github/workflows/dbt-slim-ci.yml b/.github/workflows/dbt-slim-ci.yml
new file mode 100644
index 0000000..2bcb385
--- /dev/null
+++ b/.github/workflows/dbt-slim-ci.yml
@@ -0,0 +1,78 @@
+name: dbt CI
+
+on:
+  pull_request:
+    branches:
+      - master
+
+jobs:
+  dbt-run:
+    runs-on: ubuntu-latest
+    environment: DBT_MI
+    defaults:
+      run:
+        working-directory: marketing_insights
+
+    env:
+      DBT_PRIVATE_KEY: ${{ secrets.DBT_PRIVATE_KEY }}
+      DBT_PRIVATE_KEY_PATH: /home/runner/.dbt/rsa_key.p8
+      DBT_SNOWFLAKE_ACCOUNT: ${{ secrets.DBT_SNOWFLAKE_ACCOUNT }}
+      DBT_SNOWFLAKE_ROLE: ${{ secrets.DBT_SNOWFLAKE_ROLE }}
+      DBT_SNOWFLAKE_USER: ${{ secrets.DBT_SNOWFLAKE_USER }}
+      DBT_SCHEMA: pr_${{ github.event.pull_request.number }}_ANALYTICS
+
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0   # needed so we can access master too
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.12.13"
+
+      - name: Install dbt
+        run: |
+          python -m pip install --upgrade pip
+          pip install dbt-core==1.11.8 dbt-snowflake==1.11.4
+
+      - name: Set up dbt profile + private key
+        run: |
+          mkdir -p ~/.dbt
+          cp .dbt/profiles.yml ~/.dbt/profiles.yml
+          echo "$DBT_PRIVATE_KEY" > ~/.dbt/rsa_key.p8
+          chmod 600 ~/.dbt/rsa_key.p8
+
+      - name: Install dbt deps
+        run: dbt deps
+
+      - name: Run dbt debug
+        run: dbt debug --target ci      
+
+      - name: Create state manifest from master
+        run: |
+          git checkout master
+          dbt deps
+          dbt parse --target prod
+          mkdir -p state
+          cp target/manifest.json state/manifest.json
+          git checkout -
+
+      # ---- Clone incrementals first (so is_incremental() is true) ----
+      - name: Clone impacted incremental models into PR schema
+        run: |
+          dbt clone \
+            --target ci \
+            --state ./state \
+            --defer \
+            --select "state:modified+,config.materialized:incremental,state:old"
+
+      # ---- Then slim build ----
+      - name: Slim build (modified + downstream)
+        run: |
+          dbt build \
+            --target ci \
+            --state ./state \
+            --defer \
+            --select "state:modified+"
\ No newline at end of file
diff --git a/.gitignore b/.gitignore
index b02c6d9..0096b6b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -2,6 +2,7 @@
 target/
 dbt_packages/
 logs/
+state/
 
 #Secrets
 rsa_key.p8
diff --git a/ReadMe b/ReadMe
index fa7afaa..e3396bd 100644
--- a/ReadMe
+++ b/ReadMe
@@ -28,4 +28,10 @@ using logs folder
 -- Deep dive
 using target dir and look into compile and run folders
 
-kill -9 $(lsof -t -i:8080)
\ No newline at end of file
+kill -9 $(lsof -t -i:8080)
+
+dbt compile
+copy the target/manifest.json to state/
+dbt run -m <model_name> --defer --state state
+
+dbt clone --state state
\ No newline at end of file