From d6ce6ff0811f26f5d961731f1d30a7f650db13eb Mon Sep 17 00:00:00 2001 From: database-tycoon Date: Fri, 2 Jan 2026 18:31:14 -0500 Subject: [PATCH] chore: Final polish for v0.2 release MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Changes ### Documentation - Fix docs/README.md broken links (agents/ ā†’ llm-context/) - Remove stale planning docs (NEXT_STEPS_SCHEMA_DRIFT.md, SCHEMA_DRIFT_ELIMINATION_PLAN.md) - Add v0.2 release notes - Add v0.3 roadmap with deferred features - Update docs index with v0.2 and v0.3 links ### Pre-commit - Update pre-commit-hooks to v5.0.0 - Update Black to 24.4.2 (Python 3.12.5 compatible) - Apply Black formatting to 18 files ### Code Style - Black formatting applied consistently across codebase šŸ¤– Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- .pre-commit-config.yaml | 4 +- docs/NEXT_STEPS_SCHEMA_DRIFT.md | 440 ---------------------- docs/README.md | 16 +- docs/SCHEMA_DRIFT_ELIMINATION_PLAN.md | 433 --------------------- docs/releases/v0.2/RELEASE_NOTES.md | 97 +++++ docs/releases/v0.3/ROADMAP.md | 173 +++++++++ scripts/github_issue_to_snowddl.py | 6 +- scripts/investigate_object_ownership.py | 4 +- scripts/manage_streamlit_viewer.py | 4 +- scripts/query_all_schemas.py | 16 +- src/automation/issue_parser.py | 24 +- src/automation/yaml_generator.py | 12 +- src/investigate_resource_monitors.py | 28 +- src/snowddl_core/account_objects.py | 6 +- src/snowtower_core/managers.py | 8 +- src/snowtower_snowddl/cli.py | 24 +- src/user_management/health_check.py | 4 +- src/user_management/password_generator.py | 3 +- src/user_management/yaml_handler.py | 4 +- src/verify_password.py | 4 +- tests/fixtures/config_templates.py | 2 +- tests/test_password_generation.py | 11 +- tests/test_snowddl_core.py | 6 +- tests/test_user_manager.py | 2 +- uv.lock | 2 +- 25 files changed, 383 insertions(+), 950 deletions(-) delete mode 100644 docs/NEXT_STEPS_SCHEMA_DRIFT.md delete mode 100644 docs/SCHEMA_DRIFT_ELIMINATION_PLAN.md create mode 100644 docs/releases/v0.2/RELEASE_NOTES.md create mode 100644 docs/releases/v0.3/ROADMAP.md diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 10e3b0e..722faec 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -3,7 +3,7 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.5.0 + rev: v5.0.0 # Updated from v4.5.0 (Jan 2026) hooks: - id: trailing-whitespace - id: end-of-file-fixer @@ -25,7 +25,7 @@ repos: always_run: false - repo: https://github.com/psf/black - rev: 23.12.1 + rev: 24.4.2 # Updated from 23.12.1 - stable version compatible with Python 3.12.5 hooks: - id: black exclude: ^(tests/manual/|venv/|\.venv/) diff --git a/docs/NEXT_STEPS_SCHEMA_DRIFT.md b/docs/NEXT_STEPS_SCHEMA_DRIFT.md deleted file mode 100644 index 68c19af..0000000 --- a/docs/NEXT_STEPS_SCHEMA_DRIFT.md +++ /dev/null @@ -1,440 +0,0 @@ -# Next Steps: Schema Drift Elimination Implementation - -**Branch:** `feature/eliminate-schema-drift` -**Status:** Phase 1 Complete - Ready for Testing & Validation -**Last Updated:** 2025-11-26 - ---- - -## āœ… Completed: Phase 1 - Schema Definitions - -- [x] Created schema.yaml files for all 13 databases -- [x] Built test script to compare WITH/WITHOUT schema exclusion (`test-schema-mgmt`) -- [x] Documented comprehensive 10-phase migration plan -- [x] Committed all changes to feature branch - ---- - -## šŸ”„ In Progress: Phase 2 - Validation & Testing - -### Task 1: Run Schema Management Test - -**Purpose:** Validate that SnowDDL can successfully manage SCHEMA objects without exclusion - -**Commands:** -```bash -# Ensure you're on the feature branch -git checkout feature/eliminate-schema-drift - -# Run the test (safe, read-only, no changes applied) -uv run test-schema-mgmt -``` - -**Expected Output:** -- Comparison of SnowDDL behavior with/without SCHEMA exclusion -- Count of schemas that would be created -- Count of drift warnings that would be eliminated -- Detailed logs saved to `test-output/` directory - -**Decision Point:** -- āœ… If successful ā†’ Proceed to Task 2 -- āŒ If errors ā†’ Investigate SnowDDL schema handling before proceeding - ---- - -### Task 2: Research SnowDDL Schema Grant Support - -**Purpose:** Determine if SnowDDL natively supports schema-level grants in YAML - -**Investigation Steps:** - -1. **Check SnowDDL Documentation:** - ```bash - # Search SnowDDL docs for schema grant examples - # URL: https://snowddl.readthedocs.io/ - ``` - -2. **Examine SnowDDL Source Code:** - ```bash - # Find SnowDDL installation location - python -c "import snowddl; print(snowddl.__file__)" - - # Search for schema grant handling - grep -r "GRANT.*SCHEMA" - ``` - -3. **Test Schema Grant Syntax:** - - Try adding grants directly in `snowddl/SOURCE_STRIPE/schema.yaml`: - ```yaml - STRIPE_WHY: - comment: "Source data schema" - retention_time: 1 - grants: # Does SnowDDL support this? - USAGE: - - COMPANY_USERS__B_ROLE - - DBT_STRIPE_ROLE__T_ROLE - ALL: - - DLT_STRIPE_TECH_ROLE__T_ROLE - ``` - - Then run: `uv run snowddl-plan` to see if it's recognized - -**Possible Outcomes:** - -**Scenario A: SnowDDL Supports Schema Grants āœ…** -- Update all schema.yaml files with grant configurations -- Eliminate `apply_schema_grants.py` script entirely -- **Best outcome - proceed to Phase 3** - -**Scenario B: SnowDDL Doesn't Support Schema Grants āŒ** -- Keep `apply_schema_grants.py` script for now -- Schema grants still applied separately (but no drift!) -- **Still eliminates drift warnings - proceed to Phase 3** - ---- - -### Task 3: Verify dbt Compatibility - -**Purpose:** Ensure dbt can work with pre-existing schemas (not creating them itself) - -**If you have access to dbt project:** - -1. **Locate dbt Configuration:** - ```bash - # Find dbt_project.yml (may be in separate repository) - find ~ -name "dbt_project.yml" 2>/dev/null - ``` - -2. **Check Schema Configuration:** - ```yaml - # Look for schema directives in dbt_project.yml - models: - project_name: - +schema: PROJ_STRIPE # Should reference existing schema - ``` - -3. **Verify No Dynamic Schema Creation:** - ```bash - # Search for CREATE SCHEMA commands in dbt macros - grep -r "CREATE SCHEMA" /macros/ - ``` - -**If you DON'T have access to dbt project:** -- Document assumption: dbt will use pre-existing schemas -- Plan to test in staging environment before production -- **Acceptable risk - most dbt projects work this way** - ---- - -## ā³ Pending: Phase 3 - Implementation - -### Task 4: Remove Schema Exclusion from SnowDDL CLI - -**Files to Modify:** - -All instances of `--exclude-object-types SCHEMA` must be changed: - -1. **`src/snowtower_snowddl/cli.py`** (6 locations): - - Line 220, 362, 483, 567, 646, 688 - -2. **`src/snowtower_snowddl/intelligent_apply.py`** (1 location): - - Line 196 - -**Change:** -```python -# BEFORE -"--exclude-object-types", "PIPE,STREAM,TASK,SCHEMA" - -# AFTER -"--exclude-object-types", "PIPE,STREAM,TASK" -``` - -**Testing After Changes:** -```bash -# Plan should now show schema operations -uv run snowddl-plan - -# Look for CREATE SCHEMA statements -uv run snowddl-plan | grep "CREATE SCHEMA" -``` - ---- - -### Task 5: Create Additional Schema Definitions (If Needed) - -**Check for Missing Schemas:** -```bash -# Query Snowflake to find all existing schemas -snowsql -q " -SELECT - CATALOG_NAME as DATABASE, - SCHEMA_NAME, - CREATED, - OWNER -FROM INFORMATION_SCHEMA.SCHEMATA -WHERE CATALOG_NAME NOT IN ('SNOWFLAKE', 'INFORMATION_SCHEMA') -ORDER BY CATALOG_NAME, SCHEMA_NAME; -" -``` - -**For each schema not in SnowDDL YAML:** -- Create corresponding schema.yaml entry -- Or decide to let SnowDDL drop it (if truly unmanaged) - ---- - -## ā³ Pending: Phase 4 - Testing - -### Task 6: Comprehensive Test Plan - -**Test 1: Dev Environment Validation** -```bash -# Deploy to DEV_ALICE (isolated environment) -uv run snowddl-apply - -# Verify schemas created -snowsql -q "SHOW SCHEMAS IN DATABASE DEV_ALICE;" - -# Check grants applied -snowsql -q "SHOW GRANTS ON SCHEMA DEV_ALICE.TEST_SCHEMA;" -``` - -**Test 2: dbt Integration Test** (if applicable) -```bash -# Run dbt against SnowDDL-managed schemas -cd -dbt run --select source:stripe_why -dbt run --select models/stripe/* -``` - -**Test 3: Drift Validation** -```bash -# Run plan twice - should be identical -uv run snowddl-plan > plan1.txt -uv run snowddl-plan > plan2.txt -diff plan1.txt plan2.txt # Should be empty! -``` - -**Test 4: Schema Grant Persistence** -```bash -# Apply infrastructure -uv run snowddl-apply - -# Check if grants persist (no separate script needed) -snowsql -q "SHOW GRANTS ON SCHEMA SOURCE_STRIPE.STRIPE_WHY;" -``` - ---- - -## ā³ Pending: Phase 5 - CI/CD Updates - -### Task 7: Update GitHub Actions Workflow - -**File:** `.github/workflows/merge-deploy.yml` - -**Changes Needed:** - -1. **Remove separate schema grants step:** - ```yaml - # REMOVE THIS STEP (if schema grants are native in SnowDDL) - - name: "Apply Schema Grants (Required)" - run: | - uv run apply-schema-grants - ``` - -2. **Update deployment step description:** - ```yaml - # Update description to reflect schema management - - name: "Deploy Infrastructure (includes schemas)" - run: | - uv run snowddl-apply - ``` - -3. **Remove intelligent filtering logic** (if present): - - Any drift suppression scripts - - Any REVOKE filtering tools - ---- - -## ā³ Pending: Phase 6 - Documentation Updates - -### Task 8: Update Project Documentation - -**Files to Update:** - -1. **Mark as DEPRECATED or ARCHIVE:** - - `docs/SCHEMA_GRANTS_CRITICAL.md` ā†’ Add "DEPRECATED" banner - - `docs/SCHEMA_GRANTS_WORKAROUND.md` ā†’ Move to `docs/archive/` - -2. **Update with Resolution:** - - `docs/blog/01_snowddl_schema_crisis.md` ā†’ Add epilogue section - - `docs/blog/narration/01_snowddl_schema_crisis_narration.md` ā†’ Add final chapter - -3. **Update References:** - - `README.md` ā†’ Remove schema grant workaround mentions - - `docs/ARCHITECTURE.md` ā†’ Update architecture diagrams - -4. **Create New Documentation:** - - `docs/SCHEMA_MANAGEMENT.md` ā†’ How SnowDDL manages schemas - - `docs/DBT_INTEGRATION.md` ā†’ How dbt works with SnowDDL schemas - ---- - -## ā³ Pending: Phase 7 - Production Rollout - -### Task 9: Staged Rollout Plan - -**Week 1: Dev Environment** -- [ ] Deploy to all DEV databases -- [ ] Monitor for 1 week -- [ ] Validate no issues - -**Week 2: Staging Environment** (if available) -- [ ] Deploy to staging -- [ ] Run full test suite -- [ ] Validate dbt compatibility -- [ ] Monitor for 1 week - -**Week 3: Production Deployment** -- [ ] Create backup of current state -- [ ] Schedule maintenance window -- [ ] Deploy changes to production -- [ ] Monitor for 24-48 hours -- [ ] Remove deprecated scripts if successful - -**Rollback Plan:** -```bash -# If issues occur, revert immediately -git revert -git push - -# Re-enable schema exclusion temporarily -# Re-run apply_schema_grants.py -uv run apply-schema-grants - -# Verify access restored -snowsql -q "SHOW GRANTS ON SCHEMA SOURCE_STRIPE.STRIPE_WHY;" -``` - ---- - -## ā³ Pending: Task 10 - Cleanup & Communication - -### Final Steps - -1. **Remove Deprecated Code:** - - `scripts/apply_schema_grants.py` (if no longer needed) - - Any drift filtering scripts - - Pre-commit hooks for schema grants validation - -2. **Update Team Documentation:** - - Deployment runbook - - Troubleshooting guides - - New developer onboarding - -3. **Communication:** - - Announce to team via Slack/email - - Update internal wiki/docs - - Create blog post: "How We Eliminated 15,000 Lines of Infrastructure Noise" - -4. **Merge Feature Branch:** - ```bash - git checkout main - git merge feature/eliminate-schema-drift - git push - ``` - ---- - -## šŸ“Š Success Metrics - -**Track Before/After:** - -| Metric | Before | Target | Actual | -|--------|--------|--------|--------| -| Schema drift warnings per PR | ~200-300 | 0 | TBD | -| Lines of noise per month | 15,000+ | 0 | TBD | -| PR review time | X minutes | -50% | TBD | -| Deployment failures (drift-related) | Y/month | 0 | TBD | - ---- - -## āœ… Resolved Questions - -1. **Does SnowDDL support schema-level grants in YAML?** - - Status: **YES - RESOLVED** - - Solution: Use `SCHEMA:` format in `tech_role.yaml` - - Example: - ```yaml - DBT_STRIPE_ROLE: - grants: - SCHEMA:USAGE: - - SOURCE_STRIPE.STRIPE_WHY - SCHEMA:CREATE TABLE,MODIFY: - - PROJ_STRIPE.PROJ_STRIPE - ``` - - Impact: Can eliminate drift by adding schema grants to YAML - -2. **Why did test script show different results than `uv run snowddl-plan`?** - - Status: **RESOLVED** - - Root cause: `--env-prefix SNOWFLAKE` prefixes OBJECT NAMES, not env vars - - Fix: Remove `--env-prefix`, use `-r ACCOUNTADMIN` to see all grants - - Lesson: `--env-prefix` is for environment separation (DEV__, PROD__), not connection config - -## šŸšØ Remaining Open Questions - -1. **Does dbt require CREATE SCHEMA privileges?** - - Status: Assumed NO (standard dbt practice) - - Action: Verify in Task 3 - - Impact: Low - dbt typically uses existing schemas - -2. **Are there schemas in Snowflake not in SnowDDL YAML?** - - Status: Unknown - - Action: Query Snowflake in Task 5 - - Impact: SnowDDL may try to drop unmanaged schemas - -3. **Do we have a staging environment for testing?** - - Status: Unclear - - Action: Ask user - - Impact: Affects rollout strategy (skip to production if no staging) - ---- - -## šŸ“ Related Files - -- **Plan:** [`docs/SCHEMA_DRIFT_ELIMINATION_PLAN.md`](SCHEMA_DRIFT_ELIMINATION_PLAN.md) - Full strategy -- **Test Script:** [`scripts/test_schema_management.py`](../scripts/test_schema_management.py) - Validation tool -- **Schema YAMLs:** `snowddl/*/schema.yaml` - All database schemas -- **Current Workaround:** [`docs/SCHEMA_GRANTS_CRITICAL.md`](SCHEMA_GRANTS_CRITICAL.md) - To be deprecated - ---- - -## šŸŽÆ Priority Order - -**Critical Path** (must be done in order): - -1. āœ… Create schema.yaml files (DONE) -2. šŸ”„ Run test-schema-mgmt to validate approach (NEXT) -3. šŸ”„ Research SnowDDL schema grant support (NEXT) -4. ā³ Remove SCHEMA exclusion from CLI -5. ā³ Test in dev environment -6. ā³ Deploy to production -7. ā³ Update documentation -8. ā³ Remove deprecated code - -**Can be done in parallel:** -- Verify dbt compatibility (Task 3) -- Update CI/CD workflows (Task 7) -- Prepare documentation updates (Task 8) - ---- - -**Ready to Continue?** - -Start with Task 1: -```bash -uv run test-schema-mgmt -``` - -Then review the output and proceed based on results! šŸš€ diff --git a/docs/README.md b/docs/README.md index 13632f0..37eff3a 100644 --- a/docs/README.md +++ b/docs/README.md @@ -10,7 +10,7 @@ | Fix schema permissions | [Schema Grants](guide/SCHEMA_GRANTS.md) | | Troubleshoot issues | [Troubleshooting](guide/TROUBLESHOOTING.md) | | View the changelog | [Changelog](releases/CHANGELOG.md) | -| Use LLMs with this repo | [Agent Config](agents/) | +| Use LLMs with this repo | [LLM Context](llm-context/) | --- @@ -41,14 +41,14 @@ All user-facing documentation. --- -## Agents (LLM Configuration) +## LLM Context Configuration files for using LLMs with this codebase. -- [README](agents/README.md) - Overview and quick start -- [CLAUDE.md](agents/CLAUDE.md) - Full project instructions for Claude -- [CONTEXT.md](agents/CONTEXT.md) - Domain knowledge and project context -- [PATTERNS.md](agents/PATTERNS.md) - Code patterns and conventions +- [README](llm-context/README.md) - Overview and quick start +- [CLAUDE.md](llm-context/CLAUDE.md) - Full project instructions for Claude +- [CONTEXT.md](llm-context/CONTEXT.md) - Domain knowledge and project context +- [PATTERNS.md](llm-context/PATTERNS.md) - Code patterns and conventions --- @@ -66,6 +66,8 @@ For developers and contributors. ## Releases - [Changelog](releases/CHANGELOG.md) - Version history +- [v0.2 Release](releases/v0.2/) - CI/CD & Developer Experience +- [v0.3 Roadmap](releases/v0.3/) - Upcoming features - [v0.1 Release](releases/v0.1/) - Initial release docs --- @@ -99,4 +101,4 @@ Historical documentation: [Archive](archive/) --- -**Last Updated**: November 2025 +**Last Updated**: January 2026 diff --git a/docs/SCHEMA_DRIFT_ELIMINATION_PLAN.md b/docs/SCHEMA_DRIFT_ELIMINATION_PLAN.md deleted file mode 100644 index 951c6e2..0000000 --- a/docs/SCHEMA_DRIFT_ELIMINATION_PLAN.md +++ /dev/null @@ -1,433 +0,0 @@ -# Schema Drift Elimination Plan - -## Executive Summary - -**Goal**: Eliminate schema drift warnings by having SnowDDL manage SCHEMA objects directly, while maintaining compatibility with dbt workflows. - -**Current Problem**: -- SnowDDL excludes SCHEMA objects (`--exclude-object-types SCHEMA`) -- Schema grants applied via separate `apply_schema_grants.py` script -- SnowDDL sees these grants as "drift" and suggests revoking them -- Creates 15,000+ lines of noise per month that we must manually filter - -**Proposed Solution**: -- **Remove** `--exclude-object-types SCHEMA` from SnowDDL configuration -- **Define** all schemas explicitly in SnowDDL YAML files -- **Let** dbt operate on pre-existing schemas instead of creating them -- **Eliminate** the separate `apply_schema_grants.py` script entirely - ---- - -## Current Architecture (Problematic) - -```mermaid -graph LR - A[SnowDDL Deploy] -->|Excludes SCHEMA objects| B[Infrastructure Applied] - B --> C[apply_schema_grants.py] - C -->|Manual grants| D[Schemas have permissions] - E[SnowDDL Plan] -->|Scans Snowflake| F[Detects drift] - F -->|Suggests REVOKE| G[Noise in every PR] - H[dbt Run] -->|May create schemas| I[More drift] -``` - -**Problems**: -1. Two separate systems managing schema grants (SnowDDL + script) -2. Drift detection noise in every deployment plan -3. Risk of accidentally applying REVOKEs -4. Complex CI/CD with filtering logic -5. Cognitive overhead training developers to ignore warnings - ---- - -## Target Architecture (Clean) - -```mermaid -graph LR - A[SnowDDL Deploy] -->|Manages SCHEMAS| B[Schemas Created] - B -->|With Grants| C[Infrastructure Complete] - D[dbt Run] -->|Uses existing schemas| E[Tables/Views Created] - F[SnowDDL Plan] -->|Scans Snowflake| G[No Drift Detected] - G --> H[Clean Plan Output] -``` - -**Benefits**: -1. Single source of truth (SnowDDL YAML) -2. Zero drift warnings -3. No separate grant script needed -4. Simplified CI/CD -5. Clear, actionable plan output - ---- - -## Investigation Phase - -### 1. Research SnowDDL SCHEMA Support - -**Questions to Answer**: -- How does SnowDDL handle SCHEMA objects when NOT excluded? -- Does SnowDDL support schema-level grants in YAML? -- What happens if a schema exists in Snowflake but not in YAML? -- Does SnowDDL drop unmanaged schemas or just warn? - -**Action Items**: -- [ ] Read SnowDDL source code for schema handling -- [ ] Review SnowDDL documentation for schema configuration format -- [ ] Check if SnowDDL supports grants within schema.yaml -- [ ] Determine if SnowDDL has "import existing" functionality - -**Resources**: -- SnowDDL GitHub: https://github.com/littleK0i/SnowDDL -- SnowDDL Docs: https://snowddl.readthedocs.io/ - -### 2. Test in Dev Environment - -**Safe Testing Strategy**: -```bash -# Use DEV_ALICE database for testing (isolated from production) -cd snowddl/DEV_ALICE - -# Create test schema.yaml -cat > schema.yaml < snowddl/PROJ_STRIPE/schema.yaml < -git push - -# Step 2: Re-add schema exclusion flag -# Restore --exclude-object-types SCHEMA - -# Step 3: Re-apply schema grants -uv run apply-schema-grants - -# Step 4: Verify access restored -snowsql -q "SHOW GRANTS ON SCHEMA SOURCE_STRIPE.STRIPE_WHY;" - -# Step 5: Communication -# Notify team via Slack/email -# Update incident log -``` - -**Rollback Decision Criteria**: -- dbt runs failing due to permissions -- Users unable to access schemas -- SnowDDL errors during deployment -- Unexpected schema deletions - -**Rollback Window**: 24 hours after production deployment - ---- - -## Success Metrics - -**Quantitative Metrics**: -- Schema drift warnings: 15,000/month ā†’ 0/month -- PR review time: -50% (no noise to filter) -- Deployment failures: Track before/after -- CI/CD execution time: Should remain similar - -**Qualitative Metrics**: -- Developer satisfaction survey -- Code review burden reduction -- Incident rate for permission issues -- Documentation clarity - ---- - -## Risk Assessment - -### High Risk -1. **Schema deletion** - If SnowDDL drops unmanaged schemas - - Mitigation: Test in dev first, create backups -2. **dbt breaking** - If dbt can't work with pre-existing schemas - - Mitigation: Verify dbt config, test thoroughly - -### Medium Risk -1. **Grant configuration** - If SnowDDL doesn't support schema grants - - Mitigation: Keep minimal grant script as backup -2. **Migration complexity** - Multiple schema.yaml files to create - - Mitigation: Automate discovery and generation - -### Low Risk -1. **Rollback needed** - Well-tested rollback procedure -2. **Documentation drift** - Comprehensive update plan - ---- - -## Open Questions - -1. **SnowDDL Schema Grants**: Does SnowDDL support `grants:` block within schema.yaml? -2. **SnowDDL Drop Behavior**: Will SnowDDL attempt to drop schemas not in YAML? -3. **dbt Create Behavior**: Does dbt throw errors if CREATE SCHEMA fails because schema exists? -4. **Future Grants**: Can we use database-level future grants instead of schema-level? - ---- - -## Next Steps - -**Immediate (This Week)**: -- [ ] Research SnowDDL source code for schema handling -- [ ] Test schema management in DEV_ALICE database -- [ ] Validate dbt compatibility with existing schemas - -**Short-term (Weeks 2-3)**: -- [ ] Create all missing schema.yaml files -- [ ] Implement changes in staging environment -- [ ] Run comprehensive test suite - -**Long-term (Week 4+)**: -- [ ] Production rollout with monitoring -- [ ] Remove deprecated scripts and documentation -- [ ] Write blog post: "How We Eliminated 15,000 Lines of Infrastructure Noise" - ---- - -**Last Updated**: 2025-11-22 -**Owner**: Database Tycoon Infrastructure Team -**Status**: šŸ”„ Planning Phase diff --git a/docs/releases/v0.2/RELEASE_NOTES.md b/docs/releases/v0.2/RELEASE_NOTES.md new file mode 100644 index 0000000..4452ff5 --- /dev/null +++ b/docs/releases/v0.2/RELEASE_NOTES.md @@ -0,0 +1,97 @@ +# SnowTower v0.2.0 Release Notes + +**Release Date:** January 2026 +**Focus:** CI/CD & Developer Experience + +--- + +## Highlights + +- **Automated CI/CD**: Every PR now runs 333 tests and linting automatically +- **Release Automation**: Just push a tag and GitHub creates the release with changelog +- **Claude Code Skills**: 3 focused skills replace 24 redundant agent files +- **Streamlined Contributing**: Clear workflow with CONTRIBUTING.md and PR templates + +--- + +## New Features + +### CI/CD Workflows + +| Workflow | Purpose | +|----------|---------| +| `ci.yml` | Runs lint + 333 tests on every PR | +| `release.yml` | Auto-generates releases from tags | +| `labeler.yml` | Auto-labels PRs by file type | +| `changelog.yml` | Keeps changelog updated | + +### GitHub Integration + +- **PR Template**: Standardized format for all pull requests +- **Issue Template**: Self-service new user request form +- **Auto-labeling**: PRs automatically tagged (`infrastructure`, `documentation`, `python`, etc.) +- **Branch Protection**: `main` and `v*` branches require PRs and CI passing + +### Claude Code Skills + +Replaced 24 redundant agent files with 3 focused skills: + +| Skill | Purpose | +|-------|---------| +| `snowtower-user` | End-users: access requests, connecting to Snowflake | +| `snowtower-admin` | Admins: SnowDDL operations, user management, troubleshooting | +| `snowtower-maintainer` | Project maintenance: README updates, documentation sync | + +### Documentation + +- **CONTRIBUTING.md**: Complete guide to branch strategy and PR workflow +- **Updated README**: Accurate CI/CD section with workflow diagrams +- **Reorganized docs**: `docs/agents/` renamed to `docs/llm-context/` + +--- + +## Breaking Changes + +None. This release is fully backwards compatible with v0.1. + +--- + +## Upgrade Guide + +```bash +# Pull latest changes +git pull origin main + +# Install dependencies (no changes to requirements) +uv sync + +# Verify everything works +uv run pytest +uv run snowddl-plan +``` + +--- + +## What's Next (v0.3 Roadmap) + +Features deferred from v0.2 for future consideration: + +- **Drift Detection Workflow**: Scheduled job to detect configuration drift +- **Issue ā†’ PR Automation**: Auto-generate PRs from user request issues +- **Scheduled Health Checks**: Daily/weekly automated infrastructure validation +- **Enhanced Test Coverage**: Target >80% coverage + +--- + +## Contributors + +- Database Tycoon Team +- Claude Code + +--- + +## Links + +- [Full Changelog](../CHANGELOG.md) +- [v0.2 Proposal](PROPOSAL.md) +- [GitHub Repository](https://github.com/Database-Tycoon/SnowTower) diff --git a/docs/releases/v0.3/ROADMAP.md b/docs/releases/v0.3/ROADMAP.md new file mode 100644 index 0000000..0aca2da --- /dev/null +++ b/docs/releases/v0.3/ROADMAP.md @@ -0,0 +1,173 @@ +# SnowTower v0.3 Roadmap + +**Status:** Planning +**Focus:** Advanced Automation & Quality + +--- + +## Overview + +v0.3 builds on the CI/CD foundation from v0.2 with advanced automation features and improved test coverage. + +--- + +## Deferred from v0.2 + +These features were in the v0.2 proposal but deferred: + +### 1. Drift Detection Workflow + +**Priority:** High +**Effort:** Medium + +Scheduled workflow to detect configuration drift: + +```yaml +# .github/workflows/drift-detection.yml +name: Drift Detection +on: + schedule: + - cron: '0 6 * * *' # Daily at 6 AM +jobs: + detect: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - run: uv run snowddl-plan + - name: Create issue if drift detected + if: failure() + uses: actions/github-script@v7 + # ... create issue with drift details +``` + +**Benefits:** +- Early detection of manual changes +- Audit trail of infrastructure state +- Alerts before drift becomes problematic + +--- + +### 2. Issue ā†’ PR Automation + +**Priority:** Medium +**Effort:** Medium + +Automatically generate PRs from user request issues: + +```yaml +# .github/workflows/process-user-request.yml +name: Process User Request +on: + issues: + types: [opened] +jobs: + process: + if: contains(github.event.issue.labels.*.name, 'user-request') + steps: + - run: uv run process-access-request --issue ${{ github.event.issue.number }} + - run: gh pr create --title "feat: Add user from issue #${{ github.event.issue.number }}" +``` + +**Benefits:** +- Self-service user provisioning +- Reduces admin workload +- Consistent user configuration + +--- + +### 3. Scheduled Health Checks + +**Priority:** Low +**Effort:** Low + +Weekly automated health validation: + +```yaml +# .github/workflows/health-check.yml +name: Scheduled Health Check +on: + schedule: + - cron: '0 8 * * 1' # Weekly on Monday +jobs: + health: + steps: + - run: uv run monitor-health + - run: uv run manage-costs --analyze +``` + +**Benefits:** +- Proactive issue detection +- Cost monitoring +- Security compliance checks + +--- + +## New Features for v0.3 + +### 4. Enhanced Test Coverage + +**Priority:** High +**Effort:** High + +Target: >80% test coverage (currently ~40%) + +- Integration tests for CLI commands +- End-to-end deployment tests +- Mock Snowflake connection tests + +### 5. snowddl-plan PR Comments + +**Priority:** High +**Effort:** Medium + +Post plan output directly to PR comments: + +```yaml +- name: Post plan to PR + uses: actions/github-script@v7 + with: + script: | + github.rest.issues.createComment({ + issue_number: context.issue.number, + body: `### SnowDDL Plan\n\`\`\`\n${planOutput}\n\`\`\`` + }) +``` + +**Requires:** Snowflake secrets in GitHub Actions + +### 6. PyPI Publishing + +**Priority:** Low +**Effort:** Low + +Enable `pip install snowtower`: + +```yaml +# .github/workflows/publish.yml +- uses: pypa/gh-action-pypi-publish@release/v1 + with: + password: ${{ secrets.PYPI_API_TOKEN }} +``` + +--- + +## Open Questions + +1. **Snowflake OIDC**: Should we use OIDC for GitHub Actions ā†’ Snowflake auth? +2. **Auto-apply**: Should merging PRs automatically apply to Snowflake? +3. **Multi-environment**: Support for dev/staging/prod in workflows? + +--- + +## Success Metrics + +| Metric | v0.2 | v0.3 Target | +|--------|------|-------------| +| Test coverage | ~40% | >80% | +| Drift detection | Manual | Automated daily | +| User provisioning | Semi-manual | Self-service | +| PR plan visibility | None | Auto-commented | + +--- + +*Created: January 2026* diff --git a/scripts/github_issue_to_snowddl.py b/scripts/github_issue_to_snowddl.py index 39aa8f0..661260c 100644 --- a/scripts/github_issue_to_snowddl.py +++ b/scripts/github_issue_to_snowddl.py @@ -230,9 +230,9 @@ def save_artifacts(config: GeneratedUserConfig, output_dir: Path): "username": username, "email": config.yaml_config[username].get("email"), "temporary_password": config.temp_password, - "private_key_path": str(config.private_key_path) - if config.private_key_path - else None, + "private_key_path": ( + str(config.private_key_path) if config.private_key_path else None + ), "metadata": config.metadata, } diff --git a/scripts/investigate_object_ownership.py b/scripts/investigate_object_ownership.py index de84af0..c435ee1 100644 --- a/scripts/investigate_object_ownership.py +++ b/scripts/investigate_object_ownership.py @@ -115,7 +115,9 @@ def main(): objects = cursor.fetchall() print(f"\nFound {len(objects)} objects:\n") for obj in objects: - owner_flag = "āŒ ACCOUNTADMIN" if obj[2] == "ACCOUNTADMIN" else f"āœ… {obj[2]}" + owner_flag = ( + "āŒ ACCOUNTADMIN" if obj[2] == "ACCOUNTADMIN" else f"āœ… {obj[2]}" + ) print(f" {obj[1]:12} | {obj[0]:40} | {owner_flag} | Created: {obj[3]}") # Step 2: Check who created these objects (last 7 days) diff --git a/scripts/manage_streamlit_viewer.py b/scripts/manage_streamlit_viewer.py index 68287ff..7bff31b 100644 --- a/scripts/manage_streamlit_viewer.py +++ b/scripts/manage_streamlit_viewer.py @@ -404,7 +404,9 @@ def main(): print("\nšŸ“Š Verification Summary:") print(f" Role exists: {'āœ…' if results['role_exists'] else 'āŒ'}") - print(f" Warehouse exists: {'āœ…' if results['warehouse_exists'] else 'āŒ'}") + print( + f" Warehouse exists: {'āœ…' if results['warehouse_exists'] else 'āŒ'}" + ) print( f" Permissions granted: {len(results['warehouse_grants']) + len(results['database_grants']) + len(results['account_grants'])}" ) diff --git a/scripts/query_all_schemas.py b/scripts/query_all_schemas.py index 271a9ce..7483d01 100644 --- a/scripts/query_all_schemas.py +++ b/scripts/query_all_schemas.py @@ -154,12 +154,16 @@ def query_all_schemas(): table[3], # owner f"{table[4]:,}" if table[4] else "0", # row_count f"{table[5]:,}" if table[5] else "0", # bytes - table[6].strftime("%Y-%m-%d %H:%M") - if table[6] - else "N/A", # created - table[7].strftime("%Y-%m-%d %H:%M") - if table[7] - else "N/A", # last_altered + ( + table[6].strftime("%Y-%m-%d %H:%M") + if table[6] + else "N/A" + ), # created + ( + table[7].strftime("%Y-%m-%d %H:%M") + if table[7] + else "N/A" + ), # last_altered ] ) diff --git a/src/automation/issue_parser.py b/src/automation/issue_parser.py index 63f16d7..82c30d0 100644 --- a/src/automation/issue_parser.py +++ b/src/automation/issue_parser.py @@ -130,15 +130,21 @@ def to_dict(self) -> Dict[str, Any]: "full_name": self.full_name, "email": self.email, "username": self.username, - "user_type": self.user_type.value - if isinstance(self.user_type, Enum) - else self.user_type, - "role_type": self.role_type.value - if isinstance(self.role_type, Enum) - else self.role_type, - "warehouse_size": self.warehouse_size.value - if isinstance(self.warehouse_size, Enum) - else self.warehouse_size, + "user_type": ( + self.user_type.value + if isinstance(self.user_type, Enum) + else self.user_type + ), + "role_type": ( + self.role_type.value + if isinstance(self.role_type, Enum) + else self.role_type + ), + "warehouse_size": ( + self.warehouse_size.value + if isinstance(self.warehouse_size, Enum) + else self.warehouse_size + ), "business_justification": self.business_justification, "manager_email": self.manager_email, "project_team": self.project_team, diff --git a/src/automation/yaml_generator.py b/src/automation/yaml_generator.py index 6e6501d..871acf8 100644 --- a/src/automation/yaml_generator.py +++ b/src/automation/yaml_generator.py @@ -45,9 +45,9 @@ def to_dict(self) -> Dict[str, Any]: return { "username": self.username, "yaml_config": self.yaml_config, - "private_key_path": str(self.private_key_path) - if self.private_key_path - else None, + "private_key_path": ( + str(self.private_key_path) if self.private_key_path else None + ), "temp_password": self.temp_password, "metadata": self.metadata or {}, } @@ -130,9 +130,9 @@ def generate_from_issue_data( # Build base user configuration user_config = { - "type": "SERVICE" - if parsed_data.user_type.value == "service" - else "PERSON", + "type": ( + "SERVICE" if parsed_data.user_type.value == "service" else "PERSON" + ), "first_name": first_name, "last_name": last_name, "login_name": username, diff --git a/src/investigate_resource_monitors.py b/src/investigate_resource_monitors.py index 5227b2a..c975125 100644 --- a/src/investigate_resource_monitors.py +++ b/src/investigate_resource_monitors.py @@ -84,9 +84,11 @@ def load_environment(self) -> None: "user": env_vars.get("SNOWFLAKE_USER"), "role": env_vars.get("SNOWFLAKE_ROLE"), "warehouse": env_vars.get("SNOWFLAKE_WAREHOUSE"), - "auth_method": "rsa_key" - if env_vars.get("SNOWFLAKE_PRIVATE_KEY_PATH") - else "password", + "auth_method": ( + "rsa_key" + if env_vars.get("SNOWFLAKE_PRIVATE_KEY_PATH") + else "password" + ), } self.logger.info( @@ -229,9 +231,9 @@ def test_connectivity(self) -> bool: "status": "success", "details": "Successfully connected to Snowflake", "output": stdout.strip(), - "available_connections": conn_stdout - if conn_success - else "Could not retrieve connections", + "available_connections": ( + conn_stdout if conn_success else "Could not retrieve connections" + ), } self.console.print("[green]āœ“[/green] Snowflake connectivity test passed") return True @@ -240,9 +242,9 @@ def test_connectivity(self) -> bool: "status": "failed", "details": f"Connection failed: {stderr}", "error": stderr, - "available_connections": conn_stdout - if conn_success - else "Could not retrieve connections", + "available_connections": ( + conn_stdout if conn_success else "Could not retrieve connections" + ), "troubleshooting": self._generate_auth_troubleshooting(stderr), } self.console.print("[red]āœ—[/red] Snowflake connectivity test failed") @@ -341,7 +343,9 @@ def _display_auth_troubleshooting(self, error_msg: str) -> None: """ self.console.print( - Panel(content, title="šŸšØ Authentication Troubleshooting", border_style="red") + Panel( + content, title="šŸšØ Authentication Troubleshooting", border_style="red" + ) ) def check_existing_monitors(self) -> bool: @@ -781,7 +785,9 @@ def format_human_summary(self) -> str: if self.results["safe_to_deploy"] is True: summary.append("šŸŸ¢ **APPROVED** - Deployment appears safe") elif self.results["safe_to_deploy"] == "conditional": - summary.append("šŸŸ” **CONDITIONAL** - Address warnings before deployment") + summary.append( + "šŸŸ” **CONDITIONAL** - Address warnings before deployment" + ) else: summary.append( "šŸ”“ **NOT APPROVED** - Fix critical issues before deployment" diff --git a/src/snowddl_core/account_objects.py b/src/snowddl_core/account_objects.py index 4504024..37600e5 100644 --- a/src/snowddl_core/account_objects.py +++ b/src/snowddl_core/account_objects.py @@ -583,9 +583,9 @@ def to_yaml(self) -> dict[str, Any]: if self.enable_query_acceleration: data["enable_query_acceleration"] = self.enable_query_acceleration if self.query_acceleration_max_scale_factor != 8: - data[ - "query_acceleration_max_scale_factor" - ] = self.query_acceleration_max_scale_factor + data["query_acceleration_max_scale_factor"] = ( + self.query_acceleration_max_scale_factor + ) if self.resource_constraint: data["resource_constraint"] = self.resource_constraint if self.warehouse_params: diff --git a/src/snowtower_core/managers.py b/src/snowtower_core/managers.py index 70f9ab1..c22add2 100644 --- a/src/snowtower_core/managers.py +++ b/src/snowtower_core/managers.py @@ -568,9 +568,11 @@ def get_cost_optimization_analysis(self) -> Dict[str, Any]: warehouse_analysis = { "name": warehouse.name, - "size": warehouse.size.value - if hasattr(warehouse.size, "value") - else str(warehouse.size), + "size": ( + warehouse.size.value + if hasattr(warehouse.size, "value") + else str(warehouse.size) + ), "state": warehouse.state, "auto_suspend_minutes": auto_suspend_minutes, "auto_resume": warehouse.auto_resume, diff --git a/src/snowtower_snowddl/cli.py b/src/snowtower_snowddl/cli.py index e408bef..29562a3 100644 --- a/src/snowtower_snowddl/cli.py +++ b/src/snowtower_snowddl/cli.py @@ -115,7 +115,9 @@ def validate_config(): for required_path in required_paths: file_path = config_root / required_path if file_path.exists(): - console.print(f"āœ… [green]{required_path}[/green] - Required file found") + console.print( + f"āœ… [green]{required_path}[/green] - Required file found" + ) else: error_msg = f"āŒ {required_path}: Required file missing" errors.append(error_msg) @@ -506,7 +508,9 @@ def apply(): if result.returncode == 0: console.print("āœ… [green]Apply completed successfully![/green]") - console.print("šŸ”„ [blue]Infrastructure has been updated in Snowflake[/blue]") + console.print( + "šŸ”„ [blue]Infrastructure has been updated in Snowflake[/blue]" + ) else: console.print( f"āŒ [red]Apply failed with return code {result.returncode}[/red]" @@ -618,9 +622,7 @@ def apply_user_updates(): console.print(f"Using config directory: {config_root}") # Safety confirmation - console.print( - "āš ļø [yellow]This will modify user profiles in Snowflake![/yellow]" - ) + console.print("āš ļø [yellow]This will modify user profiles in Snowflake![/yellow]") console.print( "āš ļø [yellow]Including RSA keys, names, and other personal information![/yellow]" ) @@ -711,7 +713,9 @@ def apply_user_updates(): console.print( "\\nāœ… [green]User profile updates applied successfully![/green]" ) - console.print("šŸ”„ [blue]User profiles have been updated in Snowflake[/blue]") + console.print( + "šŸ”„ [blue]User profiles have been updated in Snowflake[/blue]" + ) console.print("\\nšŸ” [yellow]Next Steps:[/yellow]") console.print(" ā€¢ Users should test their RSA key authentication") console.print(" ā€¢ Verify updated profile information in Snowflake UI") @@ -729,7 +733,9 @@ def apply_user_updates(): console.print("Installation: pip install snowddl") sys.exit(1) except Exception as e: - console.print("āŒ [red]Error applying user updates:[/red]", str(e), markup=False) + console.print( + "āŒ [red]Error applying user updates:[/red]", str(e), markup=False + ) sys.exit(1) @@ -991,7 +997,9 @@ def lint_config(): table.add_column("Count", style="magenta") table.add_column("Status", style="green") - table.add_row("Issues", str(len(issues)), "āŒ Critical" if issues else "āœ… None") + table.add_row( + "Issues", str(len(issues)), "āŒ Critical" if issues else "āœ… None" + ) table.add_row( "Suggestions", str(len(suggestions)), diff --git a/src/user_management/health_check.py b/src/user_management/health_check.py index 1342ff5..3ce0a9f 100644 --- a/src/user_management/health_check.py +++ b/src/user_management/health_check.py @@ -415,9 +415,7 @@ def print_user_table(self, results: List[HealthCheckResult], show_all: bool = Tr score_color = ( "green" if result.health_score >= 85 - else "yellow" - if result.health_score >= 50 - else "red" + else "yellow" if result.health_score >= 50 else "red" ) table.add_row( diff --git a/src/user_management/password_generator.py b/src/user_management/password_generator.py index 16dc235..7d853f4 100644 --- a/src/user_management/password_generator.py +++ b/src/user_management/password_generator.py @@ -347,7 +347,8 @@ def display_password_info(self, password_info: Dict[str, Any]) -> None: table.add_row("Length", str(password_info["length"])) table.add_row("Generated At", password_info["generated_at"]) table.add_row( - "Encryption Valid", "āœ… Yes" if password_info["encryption_valid"] else "āŒ No" + "Encryption Valid", + "āœ… Yes" if password_info["encryption_valid"] else "āŒ No", ) console.print(table) diff --git a/src/user_management/yaml_handler.py b/src/user_management/yaml_handler.py index 46be930..4d3fb2e 100644 --- a/src/user_management/yaml_handler.py +++ b/src/user_management/yaml_handler.py @@ -362,7 +362,9 @@ def restore_backup(self, backup_name: str, confirm: bool = True) -> bool: # Copy backup to main file shutil.copy2(backup_path, self.user_yaml) - console.print(f"āœ… [green]Restored configuration from {backup_name}[/green]") + console.print( + f"āœ… [green]Restored configuration from {backup_name}[/green]" + ) return True except Exception as e: diff --git a/src/verify_password.py b/src/verify_password.py index a855efc..c35cb56 100644 --- a/src/verify_password.py +++ b/src/verify_password.py @@ -334,7 +334,9 @@ def test_roundtrip(plain_password: str) -> bool: console.print("āœ… [green]Round-trip test successful![/green]") return True else: - console.print("āŒ [red]Round-trip test failed! Passwords don't match.[/red]") + console.print( + "āŒ [red]Round-trip test failed! Passwords don't match.[/red]" + ) return False except Exception as e: diff --git a/tests/fixtures/config_templates.py b/tests/fixtures/config_templates.py index d96ce9c..2341769 100644 --- a/tests/fixtures/config_templates.py +++ b/tests/fixtures/config_templates.py @@ -132,7 +132,7 @@ "name": "MISSING_TYPE", "config": { "login_name": "TEST_USER", - "email": "test@example.com" + "email": "test@example.com", # Missing required 'type' field }, "expected_error": "Missing required field: type", diff --git a/tests/test_password_generation.py b/tests/test_password_generation.py index 7309269..89ecd52 100644 --- a/tests/test_password_generation.py +++ b/tests/test_password_generation.py @@ -127,11 +127,12 @@ def setup_method(self): self.config_dir.mkdir(parents=True, exist_ok=True) # Mock components - with patch("user_management.manager.FernetEncryption") as mock_fernet, patch( - "user_management.manager.RSAKeyManager" - ) as mock_rsa, patch("user_management.manager.YAMLHandler") as mock_yaml, patch( - "user_management.manager.SnowDDLAccountManager" - ) as mock_snowddl: + with ( + patch("user_management.manager.FernetEncryption") as mock_fernet, + patch("user_management.manager.RSAKeyManager") as mock_rsa, + patch("user_management.manager.YAMLHandler") as mock_yaml, + patch("user_management.manager.SnowDDLAccountManager") as mock_snowddl, + ): mock_fernet.return_value = MagicMock() mock_rsa.return_value = MagicMock() mock_yaml.return_value = MagicMock() diff --git a/tests/test_snowddl_core.py b/tests/test_snowddl_core.py index 1705e3d..65a24e2 100644 --- a/tests/test_snowddl_core.py +++ b/tests/test_snowddl_core.py @@ -88,7 +88,7 @@ def test_user_validation_person_requires_email(self): login_name="invalid", type="PERSON", first_name="Invalid", - last_name="User" + last_name="User", # Missing email ) @@ -102,7 +102,7 @@ def test_user_validation_service_requires_rsa(self): name="SERVICE_ACCOUNT", login_name="service", type="SERVICE", - email="service@company.com" + email="service@company.com", # Missing RSA key ) @@ -442,7 +442,7 @@ def test_project_validate(self): invalid_user = User( name="INVALID", login_name="invalid", - type="PERSON" + type="PERSON", # Missing email ) project.add_user(invalid_user) diff --git a/tests/test_user_manager.py b/tests/test_user_manager.py index 5cb2928..11342f3 100644 --- a/tests/test_user_manager.py +++ b/tests/test_user_manager.py @@ -442,7 +442,7 @@ def test_validate_user_person_missing_email(self): return_value={ "type": "PERSON", "first_name": "Test", - "last_name": "User" + "last_name": "User", # Missing email } ) diff --git a/uv.lock b/uv.lock index 032585d..b3dacd7 100644 --- a/uv.lock +++ b/uv.lock @@ -2347,7 +2347,7 @@ wheels = [ [[package]] name = "snowtower" -version = "0.1.0" +version = "0.2.0" source = { editable = "." } dependencies = [ { name = "click" },