diff --git a/.github/workflows/dbt-ci.yml b/.github/workflows/dbt-ci.yml.disabled similarity index 100% rename from .github/workflows/dbt-ci.yml rename to .github/workflows/dbt-ci.yml.disabled diff --git a/.github/workflows/dbt-slim-ci.yml b/.github/workflows/dbt-slim-ci.yml new file mode 100644 index 0000000..2bcb385 --- /dev/null +++ b/.github/workflows/dbt-slim-ci.yml @@ -0,0 +1,78 @@ +name: dbt CI + +on: + pull_request: + branches: + - master + +jobs: + dbt-run: + runs-on: ubuntu-latest + environment: DBT_MI + defaults: + run: + working-directory: marketing_insights + + env: + DBT_PRIVATE_KEY: ${{ secrets.DBT_PRIVATE_KEY }} + DBT_PRIVATE_KEY_PATH: /home/runner/.dbt/rsa_key.p8 + DBT_SNOWFLAKE_ACCOUNT: ${{ secrets.DBT_SNOWFLAKE_ACCOUNT }} + DBT_SNOWFLAKE_ROLE: ${{ secrets.DBT_SNOWFLAKE_ROLE }} + DBT_SNOWFLAKE_USER: ${{ secrets.DBT_SNOWFLAKE_USER }} + DBT_SCHEMA: pr_${{ github.event.pull_request.number }}_ANALYTICS + + steps: + - name: Checkout repo + uses: actions/checkout@v3 + with: + fetch-depth: 0 # needed so we can access master too + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.12.13" + + - name: Install dbt + run: | + python -m pip install --upgrade pip + pip install dbt-core==1.11.8 dbt-snowflake==1.11.4 + + - name: Set up dbt profile + private key + run: | + mkdir -p ~/.dbt + cp .dbt/profiles.yml ~/.dbt/profiles.yml + echo "$DBT_PRIVATE_KEY" > ~/.dbt/rsa_key.p8 + chmod 600 ~/.dbt/rsa_key.p8 + + - name: Install dbt deps + run: dbt deps + + - name: Run dbt debug + run: dbt debug --target ci + + - name: Create state manifest from master + run: | + git checkout master + dbt deps + dbt parse --target prod + mkdir -p state + cp target/manifest.json state/manifest.json + git checkout - + + # ---- Clone incrementals first (so is_incremental() is true) ---- + - name: Clone impacted incremental models into PR schema + run: | + dbt clone \ + --target ci \ + --state ./state \ + --defer \ + --select "state:modified+,config.materialized:incremental,state:old" + + # ---- Then slim build ---- + - name: Slim build (modified + downstream) + run: | + dbt build \ + --target ci \ + --state ./state \ + --defer \ + --select "state:modified+" \ No newline at end of file diff --git a/.gitignore b/.gitignore index b02c6d9..0096b6b 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ target/ dbt_packages/ logs/ +state/ #Secrets rsa_key.p8 diff --git a/ReadMe b/ReadMe index fa7afaa..e3396bd 100644 --- a/ReadMe +++ b/ReadMe @@ -28,4 +28,10 @@ using logs folder -- Deep dive using target dir and look into compile and run folders -kill -9 $(lsof -t -i:8080) \ No newline at end of file +kill -9 $(lsof -t -i:8080) + +dbt compile +copy the target/manifest.json to state/ +dbt run -m --defer --state state + +dbt clone --state state \ No newline at end of file diff --git a/marketing_insights/.dbt/profiles.yml b/marketing_insights/.dbt/profiles.yml index 3fbd8c6..a9f6149 100644 --- a/marketing_insights/.dbt/profiles.yml +++ b/marketing_insights/.dbt/profiles.yml @@ -10,4 +10,14 @@ marketing_insights: type: snowflake user: "{{ env_var('DBT_SNOWFLAKE_USER') }}" warehouse: COMPUTE_WH + prod: + account: "{{ env_var('DBT_SNOWFLAKE_ACCOUNT') }}" + database: MARKETING_INSIGHTS_CI + private_key_path: "{{ env_var('DBT_PRIVATE_KEY_PATH') }}" + role: "{{ env_var('DBT_SNOWFLAKE_ROLE') }}" + schema: ANALYTICS + threads: 1 + type: snowflake + user: "{{ env_var('DBT_SNOWFLAKE_USER') }}" + warehouse: COMPUTE_WH target: ci \ No newline at end of file