diff --git a/.claude/skills/snowtower-admin/SKILL.md b/.claude/skills/snowtower-admin/SKILL.md index ad4ebe8..84864fb 100644 --- a/.claude/skills/snowtower-admin/SKILL.md +++ b/.claude/skills/snowtower-admin/SKILL.md @@ -5,162 +5,50 @@ description: Advanced skill for SnowTower infrastructure administrators. Use for # SnowTower Administrator Guide -A comprehensive skill for administrators managing Snowflake infrastructure through SnowTower. +Assumes CLAUDE.md is loaded for project context, role hierarchy, and SnowDDL knowledge. -## Who This Skill Is For - -- **Infrastructure administrators** managing SnowDDL deployments -- **Security admins** handling user provisioning and roles -- **DevOps engineers** managing CI/CD pipelines -- **On-call engineers** troubleshooting production issues - ---- - -## Quick Command Reference +## Deployment Checklist ```bash -# Essential commands -uv run snowddl-plan # Preview changes (ALWAYS run first) -uv run deploy-safe # Apply changes safely -uv run manage-users # User lifecycle management -uv run manage-warehouses # Warehouse operations -uv run manage-costs # Cost analysis -``` - ---- - -## Core Operations - -### SnowDDL Deployment Workflow - -**CRITICAL: Always use `deploy-safe`, never raw `snowddl-apply`** - -```bash -# 1. Make changes to YAML files in snowddl/ -vim snowddl/user.yaml - +# 1. Edit YAML files in snowddl/ # 2. ALWAYS preview first uv run snowddl-plan - -# 3. Review the plan output carefully -# Look for: CREATE, ALTER, DROP, GRANT, REVOKE statements - +# 3. Review output for unexpected DROP/REVOKE statements # 4. Apply using safe deployment (preserves schema grants) uv run deploy-safe ``` -**Why `deploy-safe`?** -SnowDDL excludes SCHEMA objects from management, which can cause it to revoke schema-level grants. The `deploy-safe` wrapper automatically restores these grants after every deployment, preventing dbt and other tools from losing permissions. - -### Understanding Plan Output - -``` -[APPLY] CREATE USER "NEW_USER" ← New object will be created -[APPLY] ALTER USER "EXISTING_USER" ← Object will be modified -[APPLY] DROP USER "OLD_USER" ← Object will be deleted (CAREFUL!) -[APPLY] GRANT ROLE "X" TO USER "Y" ← Permission will be added -[APPLY] REVOKE ROLE "X" FROM USER "Y" ← Permission will be removed -``` - -**Red flags to watch for:** -- Unexpected `DROP` statements -- Mass `REVOKE` statements (might be schema drift) -- Changes to admin roles (ACCOUNTADMIN, SECURITYADMIN) - ---- +**Why `deploy-safe`?** Raw `snowddl-apply` can revoke schema-level grants managed by dbt. The wrapper restores them automatically. -## User Management +**Plan output flags:** +- `CREATE` / `ALTER` - normally expected +- `DROP` - verify this is intentional +- Mass `REVOKE` - likely schema drift (see CLAUDE.md "Schema Drift Problem") +- Changes to ACCOUNTADMIN/SECURITYADMIN - red flag -### Creating a New User +## User Creation -**Option 1: Interactive wizard (recommended)** +**Option 1: Interactive wizard** (recommended) ```bash uv run manage-users create ``` -**Option 2: Edit YAML directly** -```yaml -# snowddl/user.yaml -NEW_USER: - comment: "Data Analyst - Analytics Team" - type: PERSON - default_role: ANALYST_ROLE__B_ROLE - default_warehouse: MAIN_WAREHOUSE - email: user@company.com - authentication: - password: !decrypt | - gAAAAABl...encrypted... - rsa_public_key: | - -----BEGIN PUBLIC KEY----- - MIIBIjAN... - -----END PUBLIC KEY----- -``` +**Option 2: Edit YAML directly** in `snowddl/user.yaml` **Option 3: Non-interactive** ```bash -uv run manage-users create \ - --first-name Jane \ - --last-name Smith \ - --email jane@company.com \ - --role ANALYST_ROLE +uv run manage-users create --first-name Jane --last-name Smith --email jane@company.com --role ANALYST_ROLE ``` -### User Types - -| Type | Use For | MFA Required | Network Policy | -|------|---------|--------------|----------------| -| `PERSON` | Human users | Yes (by 2026) | Applied | -| `SERVICE` | Service accounts | No | Not applied | - -### Encrypting Passwords - +**Encrypt passwords:** ```bash -# Generate Fernet key (one-time setup) -uv run util-generate-key - -# Encrypt a password uv run snowddl-encrypt "MySecurePassword123!" -# Output: gAAAAABl... - -# Use in YAML with !decrypt tag -authentication: - password: !decrypt | - gAAAAABl...encrypted_output... -``` - -### RSA Key Setup for Users - -```bash -# Generate keys for a user -uv run generate-rsa-batch --users NEW_USER - -# Or manually: -openssl genrsa 2048 | openssl pkcs8 -topk8 -inform PEM -nocrypt -out user_key.p8 -openssl rsa -in user_key.p8 -pubout -out user_key.pub - -# Add public key to user.yaml -cat user_key.pub -``` - ---- - -## Role Hierarchy - -### SnowDDL Role Naming Convention - -``` -ROLE_NAME__B_ROLE → Business role (assigned to users) -ROLE_NAME__T_ROLE → Technical role (assigned to business roles) -DB__SCHEMA__S_ROLE → Schema role (auto-created by SnowDDL) +# Use output in YAML with !decrypt tag ``` -### Role Assignment Flow +**For service accounts:** See [Service Account Pattern](../../.claude/patterns/SERVICE_ACCOUNT_CREATION_PATTERN.md) -``` -User → Business Role (__B_ROLE) → Technical Roles (__T_ROLE) → Permissions -``` - -### Creating Roles +## Role Management **Business Role** (`snowddl/business_role.yaml`): ```yaml @@ -168,74 +56,45 @@ ANALYST_ROLE: comment: "Business analysts with read access" tech_roles: - STRIPE_READER_ROLE - - ANALYTICS_READER_ROLE warehouse_usage: - MAIN_WAREHOUSE schema_read: - PROJ_STRIPE.ANALYTICS ``` -**Technical Role** (`snowddl/tech_role.yaml`): -```yaml -STRIPE_READER_ROLE: - grants: - DATABASE:USAGE: - - SOURCE_STRIPE - - PROJ_STRIPE - SCHEMA:USAGE: - - SOURCE_STRIPE.STRIPE_WHY - - PROJ_STRIPE.PROJ_STRIPE - TABLE:SELECT: - - SOURCE_STRIPE.STRIPE_WHY.* -``` +**Technical Role** (`snowddl/tech_role.yaml`): See grant format in CLAUDE.md "SnowDDL Object Type Reference". ---- +## Troubleshooting Quick Fixes -## Database & Schema Management +| Problem | Fix | +|---------|-----| +| Schema grant drift (mass REVOKEs) | Use `uv run deploy-safe` instead of raw apply | +| "Object does not exist" | Ensure using `-r ACCOUNTADMIN` | +| "Insufficient privileges" | Check `SNOWFLAKE_ROLE=ACCOUNTADMIN` in `.env` | +| Exit code 8 | Not an error - means changes were applied | +| User locked out | `ALTER USER USERNAME SET MINS_TO_UNLOCK = 0;` | -### Creating a Database +## Emergency Procedures +**Rollback last deployment:** ```bash -# Create directory -mkdir snowddl/MY_NEW_DB - -# Add params.yaml -cat > snowddl/MY_NEW_DB/params.yaml << 'EOF' -comment: "New database for analytics project" -is_transient: false -EOF - -# Deploy -uv run snowddl-plan +git checkout HEAD~1 -- snowddl/ uv run deploy-safe ``` -### Creating a Schema - +**Service account reset:** ```bash -# Create schema directory -mkdir snowddl/MY_DB/MY_SCHEMA - -# Add params.yaml -cat > snowddl/MY_DB/MY_SCHEMA/params.yaml << 'EOF' -comment: "Schema for raw data ingestion" -is_transient: false -is_sandbox: false -EOF +uv run generate-rsa-batch --users SNOWDDL --force +snow sql -q "ALTER USER SNOWDDL SET RSA_PUBLIC_KEY='...'" +gh secret set SNOWFLAKE_PRIVATE_KEY < new_key.p8 ``` -### Schema Types - -| Parameter | Effect | -|-----------|--------| -| `is_transient: true` | No Time Travel, no Fail-safe | -| `is_sandbox: true` | Creates as TRANSIENT schema | - ---- - -## CI/CD Operations +**MFA compliance check:** +```bash +uv run manage-security --check-mfa +``` -### GitHub Actions Workflows +## CI/CD Workflows | Workflow | Trigger | Purpose | |----------|---------|---------| @@ -243,233 +102,3 @@ EOF | `release.yml` | Tags `v*` | Create GitHub release | | `labeler.yml` | PRs | Auto-label by file type | | `changelog.yml` | Push to main | Update changelog | - -### Making Infrastructure Changes via PR - -```bash -# 1. Create feature branch -git checkout v0.2 -git checkout -b feature/add-new-user - -# 2. Make YAML changes -vim snowddl/user.yaml - -# 3. Validate locally -uv run pre-commit run --all-files -uv run pytest - -# 4. Commit and push -git add . -git commit -m "feat: Add new user JANE_DOE" -git push -u origin feature/add-new-user - -# 5. Create PR -gh pr create --base v0.2 - -# 6. CI runs automatically, merge after approval -``` - -### Release Process - -```bash -# After all PRs merged to v0.2 -git checkout main -git pull -git merge v0.2 -git tag v0.2.0 -git push origin main --tags -# Release workflow creates GitHub release automatically -``` - ---- - -## Troubleshooting - -### Schema Grant Drift - -**Symptom:** Plan shows hundreds of REVOKE statements for schema grants - -**Cause:** SnowDDL doesn't manage SCHEMA objects directly; grants from dbt or other tools appear as drift - -**Solution:** -```bash -# Always use deploy-safe which auto-applies schema grants -uv run deploy-safe - -# Or manually apply schema grants -uv run apply-schema-grants -``` - -### Authentication Failures - -```bash -# Diagnose authentication issues -uv run util-diagnose-auth - -# Fix common auth problems -uv run util-fix-auth - -# Check specific user -snow sql -q "DESCRIBE USER USERNAME" -``` - -### User Locked Out - -```sql --- Check user status -SHOW USERS LIKE 'USERNAME'; - --- Unlock user (as ACCOUNTADMIN) -ALTER USER USERNAME SET MINS_TO_UNLOCK = 0; - --- Check login history -SELECT * FROM SNOWFLAKE.ACCOUNT_USAGE.LOGIN_HISTORY -WHERE USER_NAME = 'USERNAME' -ORDER BY EVENT_TIMESTAMP DESC -LIMIT 10; -``` - -### SnowDDL Errors - -**"Object does not exist"** -- Run with ACCOUNTADMIN: `uv run snowddl-plan` uses `-r ACCOUNTADMIN` -- Check object was created in correct database/schema - -**"Insufficient privileges"** -- Verify SNOWFLAKE_ROLE is set to ACCOUNTADMIN -- Check the service account has required permissions - -**Exit code 8** -- Means "changes applied successfully" -- Not an error, SnowDDL uses this to indicate modifications were made - ---- - -## Security Operations - -### Network Policies - -```yaml -# snowddl/network_policy.yaml -corporate_network_policy: - allowed_ip_list: - - 192.0.2.0/24 - - 10.0.0.0/8 - comment: "Corporate network access only" -``` - -### MFA Compliance - -**Deadline:** March 2026 for all human users - -```bash -# Check MFA status -uv run manage-security --check-mfa - -# List users without MFA -snow sql -q " - SELECT name, has_mfa_registered - FROM SNOWFLAKE.ACCOUNT_USAGE.USERS - WHERE type = 'PERSON' AND NOT has_mfa_registered -" -``` - -### Emergency Access - -The `STEPHEN_RECOVERY` account is configured without network policy for emergency access: -- Use only when primary access methods fail -- Requires password authentication -- Document all usage - ---- - -## Cost Management - -```bash -# Analyze costs -uv run manage-costs --analyze - -# Check warehouse usage -uv run manage-warehouses --status - -# Suspend all warehouses (emergency) -uv run manage-warehouses --suspend-all -``` - -### Warehouse Configuration - -```yaml -# snowddl/warehouse.yaml -MAIN_WAREHOUSE: - size: XSMALL - auto_suspend: 60 # seconds - auto_resume: true - min_cluster_count: 1 - max_cluster_count: 1 - resource_monitor: main_monitor -``` - ---- - -## Key File Locations - -| File | Purpose | -|------|---------| -| `snowddl/user.yaml` | User accounts | -| `snowddl/business_role.yaml` | Business roles | -| `snowddl/tech_role.yaml` | Technical roles with grants | -| `snowddl/warehouse.yaml` | Warehouse configuration | -| `snowddl/*_policy.yaml` | Security policies | -| `snowddl/{DB}/params.yaml` | Database configuration | -| `snowddl/{DB}/{SCHEMA}/params.yaml` | Schema configuration | - ---- - -## Environment Variables - -Required in `.env`: -```bash -SNOWFLAKE_ACCOUNT=your_account -SNOWFLAKE_USER=SNOWDDL -SNOWFLAKE_ROLE=ACCOUNTADMIN -SNOWFLAKE_WAREHOUSE=ADMIN -SNOWFLAKE_PRIVATE_KEY_PATH=~/.ssh/snowflake_rsa_key.p8 -SNOWFLAKE_CONFIG_FERNET_KEYS=your_fernet_key -``` - ---- - -## Emergency Procedures - -### Rollback Last Deployment - -```bash -# Revert YAML changes -git checkout HEAD~1 -- snowddl/ - -# Re-apply -uv run deploy-safe -``` - -### Complete Service Account Reset - -```bash -# Regenerate RSA keys -uv run generate-rsa-batch --users SNOWDDL --force - -# Update in Snowflake -snow sql -q "ALTER USER SNOWDDL SET RSA_PUBLIC_KEY='...'" - -# Update GitHub secrets -gh secret set SNOWFLAKE_PRIVATE_KEY < new_key.p8 -``` - -### Health Check - -```bash -# Quick health check -uv run monitor-health - -# Full system audit -uv run manage-security --full-audit -``` diff --git a/.claude/skills/snowtower-developer/SKILL.md b/.claude/skills/snowtower-developer/SKILL.md new file mode 100644 index 0000000..875e868 --- /dev/null +++ b/.claude/skills/snowtower-developer/SKILL.md @@ -0,0 +1,116 @@ +--- +name: snowtower-developer +description: Comprehensive skill for SnowTower contributors and developers. Use when contributing code, fixing bugs, adding features, writing tests, or developing new functionality. Triggers on mentions of development, coding, testing, PR creation, bug fixes, or feature implementation. +--- + +# SnowTower Developer Guide + +Assumes CLAUDE.md is loaded for project context and command patterns. + +## Setup + +```bash +# 1. Fork and clone +git clone https://github.com/YOUR-USERNAME/snowtower-public.git +cd snowtower-public + +# 2. Install UV (if needed) +curl -LsSf https://astral.sh/uv/install.sh | sh + +# 3. Install dependencies +uv sync --all-extras --dev + +# 4. Set up pre-commit hooks +uv run pre-commit install + +# 5. Create .env (mock credentials fine for tests) +cp .env.example .env + +# 6. Verify +uv run pytest -v +uv run pre-commit run --all-files +``` + +## Adding a New UV Command (4-Step Pattern) + +See CLAUDE.md "Creating New Commands" for the template. Summary: + +1. **Create script** in `scripts/my_script.py` - `load_dotenv()` MUST be first, use argparse +2. **Add wrapper** in `src/management_cli.py` +3. **Register** in `pyproject.toml` under `[project.scripts]` +4. **Test**: `uv sync && uv run my-command --help` + +## Running Tests + +```bash +uv run pytest -v # All tests +uv run pytest tests/test_foo.py -v # Specific file +uv run pytest --cov=src # With coverage +uv run pytest -n auto # Parallel (faster) +uv run pytest --lf # Only last-failed +uv run pytest -vv --tb=long # Detailed failure output +uv run pytest --pdb # Drop into debugger on failure +``` + +**Mocking Snowflake** (required for unit tests - no real connection needed): +```python +@patch('module.SnowflakeClient') +def test_with_mock(self, mock_client): + mock_client.return_value.execute.return_value = [] + # test implementation +``` + +## PR Workflow + +```bash +# 1. Branch from release branch +git checkout v0.2 && git pull +git checkout -b feature/my-feature + +# 2. Develop + test +uv run pre-commit run --all-files +uv run pytest -v + +# 3. Commit (conventional commits) +git commit -m "feat: Add my feature" + +# 4. Push + create PR +git push -u origin feature/my-feature +gh pr create --base v0.2 +``` + +**Commit types:** `feat:`, `fix:`, `docs:`, `test:`, `refactor:`, `chore:`, `ci:` + +## PR Checklist + +- [ ] Tests pass: `uv run pytest -v` +- [ ] Pre-commit passes: `uv run pre-commit run --all-files` +- [ ] New features have tests +- [ ] No secrets in code +- [ ] Commit messages follow convention + +## Pre-commit Hooks + +Runs automatically on commit: Black formatting, YAML validation, trailing whitespace, secrets detection, file size limits. + +```bash +uv run pre-commit run --all-files # Manual run +uv run pre-commit run --files src/my_file.py # Specific files +``` + +## Common Issues + +| Problem | Fix | +|---------|-----| +| Import errors | `uv sync --all-extras --dev` | +| Tests need Snowflake | Tests use mocks - check mock setup | +| Pre-commit failing | Run `uv run pre-commit run --all-files`, commit fixes | + +## Useful Commands + +```bash +uv run --help # See all UV commands +uv tree # Check dependencies +uv lock --upgrade # Update dependencies +uv run python # Interactive Python with project imports +``` diff --git a/.claude/skills/snowtower-maintainer/PROJECT_STRUCTURE.md b/.claude/skills/snowtower-maintainer/PROJECT_STRUCTURE.md deleted file mode 100644 index 48fd208..0000000 --- a/.claude/skills/snowtower-maintainer/PROJECT_STRUCTURE.md +++ /dev/null @@ -1,71 +0,0 @@ -# SnowTower Project Structure Reference - -Quick reference for the maintainer skill. - -## Directory Layout - -``` -snowtower/ -├── .claude/ # Claude Code configuration -│ ├── skills/ # Skills (model-invoked capabilities) -│ │ └── snowtower-maintainer/ -│ ├── agents/ # Agent definitions -│ └── patterns/ # Reusable patterns -├── .github/ # GitHub configuration -│ ├── workflows/ # CI/CD workflows -│ │ ├── ci.yml # PR validation -│ │ ├── labeler.yml # Auto-labeling -│ │ ├── changelog.yml # Changelog generation -│ │ └── release.yml # Release automation -│ ├── ISSUE_TEMPLATE/ # Issue templates -│ └── PULL_REQUEST_TEMPLATE.md -├── snowddl/ # SnowDDL YAML configurations -│ ├── user.yaml # User accounts -│ ├── business_role.yaml # Business roles -│ ├── tech_role.yaml # Technical roles -│ ├── warehouse.yaml # Warehouses -│ ├── *_policy.yaml # Security policies -│ └── {DATABASE}/ # Database-specific configs -├── src/ # Python source code -│ ├── snowddl_core/ # OOP framework -│ ├── user_management/ # User lifecycle -│ ├── web/ # Streamlit apps -│ └── management_cli.py # CLI entry points -├── scripts/ # Standalone scripts -├── tests/ # Test suite -├── docs/ # Documentation -│ ├── guide/ # User guides -│ ├── agents/ # Agent documentation -│ └── releases/ # Release documentation -├── pyproject.toml # Project configuration -├── README.md # Main documentation -└── CLAUDE.md # Claude Code instructions -``` - -## Key Files to Monitor - -| File | Purpose | Update Frequency | -|------|---------|------------------| -| `README.md` | Main project documentation | After features/releases | -| `CLAUDE.md` | Claude Code instructions | When patterns change | -| `pyproject.toml` | Commands and dependencies | When adding commands | -| `.claude/agents/*.md` | Agent definitions | When capabilities change | -| `docs/guide/*.md` | User guides | When workflows change | - -## Current Statistics (Update These) - -```yaml -# Last updated: 2024-XX-XX -users: ~13 -databases: ~6 -warehouses: ~8 -agents: ~20+ -workflows: 4 -``` - -## Maintenance Schedule - -- **Weekly**: Check for broken links, outdated statistics -- **Per Release**: Full documentation audit -- **Per Feature**: Update relevant docs and README -- **Quarterly**: Agent consolidation review diff --git a/.claude/skills/snowtower-maintainer/SKILL.md b/.claude/skills/snowtower-maintainer/SKILL.md index 79a2a5d..d3cbc48 100644 --- a/.claude/skills/snowtower-maintainer/SKILL.md +++ b/.claude/skills/snowtower-maintainer/SKILL.md @@ -5,201 +5,81 @@ description: Maintains SnowTower project documentation, README, and Claude confi # SnowTower Project Maintainer -A specialized skill for maintaining the SnowTower project's documentation, README, and Claude Code configuration. +Assumes CLAUDE.md is loaded for project context. -## Core Responsibilities +## README Audit Checklist -### 1. README Maintenance - -Keep `README.md` accurate and current: - -- **Version badges**: Ensure CI/CD badges point to correct workflows -- **Command references**: Verify all `uv run` commands are valid -- **Architecture diagrams**: Keep mermaid diagrams in sync with actual structure -- **Statistics**: Update user counts, database counts, warehouse counts -- **Links**: Verify all internal links resolve correctly - -**Audit checklist:** -```bash -# Verify commands mentioned in README actually exist -uv run --help | grep -E "snowddl-plan|deploy-safe|manage-users" - -# Check workflow badge URLs match actual workflow files -ls .github/workflows/ - -# Verify documentation links -find docs/ -name "*.md" | head -20 -``` - -### 2. Claude Folder Maintenance - -Maintain `.claude/` organization: - -``` -.claude/ -├── skills/ # Claude Code skills (like this one) -├── agents/ # Agent definitions for task delegation -├── patterns/ # Reusable patterns and templates -└── settings.local.json -``` - -**Agent audit tasks:** -- Remove duplicate or redundant agents -- Consolidate agents with overlapping purposes -- Update agent descriptions to match current capabilities -- Ensure agents reference correct file paths - -**Pattern audit tasks:** -- Verify patterns match current project conventions -- Update code examples in patterns -- Remove outdated patterns - -### 3. Documentation Sync +1. **Verify commands** mentioned in README exist: + ```bash + uv run --help | grep -E "snowddl-plan|deploy-safe|manage-users" + ``` +2. **Check workflow badges** match `.github/workflows/` +3. **Verify documentation links** resolve to existing files +4. **Update statistics** (users, databases, warehouses): + ```bash + echo "Users: $(grep -c '^ [A-Z]' snowddl/user.yaml 2>/dev/null || echo N/A)" + echo "Databases: $(ls -d snowddl/*/ 2>/dev/null | grep -v __pycache__ | wc -l)" + echo "Warehouses: $(grep -c '^ [A-Z]' snowddl/warehouse.yaml 2>/dev/null || echo N/A)" + ``` -Ensure docs reflect actual project state: +## Documentation Sync Table | Doc File | Should Match | |----------|--------------| | `docs/guide/MANAGEMENT_COMMANDS.md` | `pyproject.toml` scripts | | `docs/guide/QUICKSTART.md` | Current setup process | | `docs/guide/SCHEMA_GRANTS.md` | Current grant handling | -| Agent files in `.claude/agents/` | Available functionality | - -## Maintenance Procedures +| `README.md` | Actual project capabilities | -### Quick Health Check +## Quick Health Check ```bash -# 1. Verify project structure -ls -la snowddl/ src/ scripts/ docs/ - -# 2. Check available commands -uv run --help - -# 3. Verify tests pass -uv run pytest --co -q | tail -5 - -# 4. Check pre-commit status -uv run pre-commit run --all-files +ls -la snowddl/ src/ scripts/ docs/ # Verify structure +uv run --help # Check commands +uv run pytest --co -q | tail -5 # Verify tests collect +uv run pre-commit run --all-files # Check formatting ``` -### README Update Workflow +## Release Planning -1. **Gather current state:** +1. **Check open issues:** ```bash - # Count configured users - grep -c "^ [A-Z]" snowddl/user.yaml - - # Count databases - ls -d snowddl/*/ | grep -v __pycache__ | wc -l - - # List warehouses - grep "^ [A-Z]" snowddl/warehouse.yaml + gh issue list --state open --label P1 + gh issue list --state open ``` - -2. **Verify commands:** +2. **Review completed work since last release:** ```bash - # Extract commands from pyproject.toml - grep -A1 "\[project.scripts\]" pyproject.toml + git log v0.X..HEAD --oneline ``` - -3. **Update statistics section** in README with current counts - -4. **Verify all links** resolve to existing files - -### Agent Consolidation - -When auditing `.claude/agents/`: - -1. **List all agents:** +3. **Create version branch:** ```bash - ls .claude/agents/*.md + git checkout main && git pull + git checkout -b v0.X + git push -u origin v0.X + ``` +4. **Pre-release checklist:** + - [ ] `uv run pytest` passes + - [ ] CHANGELOG.md updated + - [ ] README current (badges, commands, stats) + - [ ] Documentation links valid +5. **Release:** + ```bash + gh release create vX.Y --title "vX.Y - Title" --notes "..." ``` -2. **Identify overlaps:** Look for agents with similar purposes - -3. **Consolidation criteria:** - - Merge agents that serve the same domain - - Keep agents with distinct, valuable roles - - Remove agents that duplicate built-in capabilities - -4. **Update references:** After consolidation, update any docs referencing removed agents - -### Self-Maintenance - -This skill should maintain itself by: - -1. Keeping this SKILL.md up to date with project changes -2. Adding new maintenance procedures as project evolves -3. Updating file paths when project structure changes -4. Documenting new patterns discovered during maintenance - -## Common Maintenance Tasks - -### Task: Update README Statistics - -```markdown -### Status & Metrics - -- **Active Users**: [COUNT] configured users with MFA -- **Databases**: [COUNT] production databases managed -- **Warehouses**: [COUNT] warehouses with auto-suspend -``` - -Update these by running: -```bash -echo "Users: $(grep -c '^ [A-Z]' snowddl/user.yaml)" -echo "Databases: $(ls -d snowddl/*/ 2>/dev/null | grep -v __pycache__ | wc -l)" -echo "Warehouses: $(grep -c '^ [A-Z]' snowddl/warehouse.yaml)" -``` - -### Task: Verify Workflow Badges - -Check that README badges match actual workflows: -```bash -# List workflows -ls .github/workflows/ +## .claude/ Directory Structure -# Verify badge URLs in README reference these files -grep "actions/workflows" README.md ``` - -### Task: Audit Agent Definitions - -```bash -# List agents and their purposes -for f in .claude/agents/*.md; do - echo "=== $f ===" - head -5 "$f" - echo -done +.claude/ +├── patterns/ # Reusable patterns (e.g., SERVICE_ACCOUNT_CREATION_PATTERN.md) +├── skills/ # Claude Code skills (4 task-specific guides) +└── settings.local.json ``` -### Task: Clean Up Obsolete Content - -Remove references to: -- Deleted files or directories -- Deprecated commands -- Old workflow names -- Removed features - -## Integration with Project - -This skill works with: - -- **CI/CD workflows**: `.github/workflows/` -- **SnowDDL configs**: `snowddl/*.yaml` -- **Python tooling**: `src/`, `scripts/` -- **Documentation**: `docs/` -- **Claude config**: `.claude/` - ## When to Trigger -Invoke this skill when: -- User asks to "update the README" -- User mentions "documentation maintenance" -- User wants to "audit the .claude folder" -- User asks about "project documentation" -- After significant feature additions +- User asks to update README or documentation - Before releases to ensure docs are current +- After significant feature additions - When onboarding new contributors +- User asks to plan a new release version diff --git a/.claude/skills/snowtower-user/SKILL.md b/.claude/skills/snowtower-user/SKILL.md index 80fa3d1..636bb0d 100644 --- a/.claude/skills/snowtower-user/SKILL.md +++ b/.claude/skills/snowtower-user/SKILL.md @@ -5,86 +5,37 @@ description: Helps end-users get Snowflake access and use the platform. Use when # SnowTower End-User Guide -A skill for helping end-users navigate the SnowTower platform to get Snowflake access and start working with data. +Assumes CLAUDE.md is loaded for project context. -## Who This Skill Is For +## Getting Access (3 Steps) -- **Data analysts** who need to query Snowflake data -- **Data scientists** who need database access for analysis -- **Engineers** who need to connect applications to Snowflake -- **New team members** requesting their first Snowflake account - -## Quick Reference - -### Getting Access (3 Steps) - -``` -Step 1: Generate RSA Keys → Step 2: Submit Request → Step 3: Connect - (on your machine) (GitHub issue) (after approval) -``` - ---- - -## Step 1: Generate Your RSA Keys - -**You MUST do this BEFORE requesting access.** +### Step 1: Generate RSA Keys ```bash -# Generate RSA key pair (run on your local machine) +# Generate key pair openssl genrsa 2048 | openssl pkcs8 -topk8 -inform PEM -nocrypt -out ~/.ssh/snowflake_rsa_key.p8 openssl rsa -in ~/.ssh/snowflake_rsa_key.p8 -pubout -out ~/.ssh/snowflake_rsa_key.pub -# Secure your private key (IMPORTANT!) +# Secure private key chmod 400 ~/.ssh/snowflake_rsa_key.p8 -# Display your PUBLIC key (copy this for the access request) +# Copy public key for access request cat ~/.ssh/snowflake_rsa_key.pub ``` -**Output looks like:** -``` ------BEGIN PUBLIC KEY----- -MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA... -...many lines of characters... ------END PUBLIC KEY----- -``` - -### Key Security Rules - -| Key Type | File | Share? | -|----------|------|--------| -| **Private key** | `~/.ssh/snowflake_rsa_key.p8` | **NEVER share this** | -| **Public key** | `~/.ssh/snowflake_rsa_key.pub` | Safe to share | - ---- +**Private key** (`~/.ssh/snowflake_rsa_key.p8`) - NEVER share. **Public key** (`.pub`) - safe to share. -## Step 2: Request Access +### Step 2: Submit Access Request -1. Go to the **[Access Request Form](../../issues/new/choose)** +1. Go to the [Access Request Form](../../issues/new/choose) 2. Select "New User Request" -3. Fill in your details: - - Full name - - Email address - - Team/department - - Reason for access - - **Paste your PUBLIC key** (from Step 1) -4. Submit the form - -**Typical approval time:** 3-5 business days - ---- - -## Step 3: Connect to Snowflake - -After your account is approved, you'll receive: -- Your **username** (usually FIRSTNAME_LASTNAME) -- The **account identifier** -- Your **default role** and **warehouse** +3. Fill in: name, email, team, reason, **paste public key** +4. Typical approval: 3-5 business days -### Using Snow CLI (Recommended) +### Step 3: Connect to Snowflake +**Snow CLI (recommended):** ```bash -# Add your connection snow connection add \ --connection-name prod \ --account YOUR_ACCOUNT \ @@ -92,163 +43,39 @@ snow connection add \ --authenticator SNOWFLAKE_JWT \ --private-key-path ~/.ssh/snowflake_rsa_key.p8 -# Test the connection snow sql -c prod -q "SELECT CURRENT_USER(), CURRENT_ROLE()" ``` -### Using Python - +**Python:** ```python import snowflake.connector - conn = snowflake.connector.connect( account='YOUR_ACCOUNT', user='YOUR_USERNAME', - private_key_file_pwd=None, private_key_file='~/.ssh/snowflake_rsa_key.p8', warehouse='MAIN_WAREHOUSE', role='YOUR_ROLE' ) ``` -### Using the Snowflake Web UI - -1. Go to your organization's Snowflake URL -2. Enter your username -3. Use the **password provided by IT** (not your RSA key) -4. Enable MFA when prompted - ---- - -## What You Get After Approval - -### Your Default Role - -New users typically receive a role like `SNOWTOWER_USERS__T_ROLE` which grants: -- Read access to shared production data -- Access to common warehouses -- Ability to create objects in your personal database - -### Your Personal Database - -You get your own database: `DEV_YOURNAME` - -```sql --- Switch to your database -USE DATABASE DEV_YOURNAME; - --- Create schemas and tables freely -CREATE SCHEMA my_analysis; -CREATE TABLE my_analysis.test_data (id INT, value VARCHAR); -``` - -### Your Default Warehouse - -Usually `MAIN_WAREHOUSE`: -- Auto-suspends after 60 seconds of inactivity -- X-Small size by default -- Shared resource - be mindful of heavy queries - ---- - ## First Session Checklist ```sql --- 1. Check your current context SELECT CURRENT_USER(), CURRENT_ROLE(), CURRENT_WAREHOUSE(); - --- 2. See what databases you can access SHOW DATABASES; - --- 3. See what roles you have SHOW ROLES; - --- 4. Switch to your dev database USE DATABASE DEV_YOURNAME; - --- 5. Create your first schema CREATE SCHEMA IF NOT EXISTS sandbox; -USE SCHEMA sandbox; - --- 6. Test creating a table -CREATE TABLE test (id INT); -INSERT INTO test VALUES (1), (2), (3); -SELECT * FROM test; -DROP TABLE test; -``` - ---- - -## Common Issues & Solutions - -### "Authentication failed" - -**Cause:** RSA key mismatch or incorrect setup - -**Solution:** -```bash -# Verify your private key is readable -ls -la ~/.ssh/snowflake_rsa_key.p8 - -# Check permissions (should be 400 or 600) -chmod 400 ~/.ssh/snowflake_rsa_key.p8 - -# Verify the public key matches what was submitted -cat ~/.ssh/snowflake_rsa_key.pub -``` - -### "Insufficient privileges" - -**Cause:** You don't have access to that object - -**Solution:** -- Check you're using the correct role: `SELECT CURRENT_ROLE();` -- Request additional access if needed via GitHub issue - -### "Warehouse is suspended" - -**Cause:** Warehouse auto-suspended to save costs - -**Solution:** -```sql --- Just run a query - it auto-resumes -SELECT 1; ``` -### "Cannot connect to Snowflake" +## Troubleshooting -**Checklist:** -1. Is your account approved? (Check the GitHub issue) -2. Is the account identifier correct? -3. Is your private key path correct? -4. Are you on the corporate network / VPN if required? - ---- +| Problem | Fix | +|---------|-----| +| Authentication failed | Check key permissions: `chmod 400 ~/.ssh/snowflake_rsa_key.p8` | +| Insufficient privileges | Verify role: `SELECT CURRENT_ROLE();` - request more access via GitHub issue | +| Cannot connect | Check: account approved? Account ID correct? Key path correct? On VPN? | ## Getting More Access -Need access to additional databases, schemas, or roles? - -1. Open a new GitHub issue -2. Specify exactly what you need access to -3. Include business justification -4. Your request will be reviewed by an admin - ---- - -## Two Authentication Methods - -| Method | Use For | How | -|--------|---------|-----| -| **RSA Key** | CLI, scripts, applications | Private key file | -| **Password** | Web UI only | Provided by IT | - -**Best Practice:** Always use RSA key authentication for programmatic access. Only use password for the web interface. - ---- - -## Need Help? - -- **Access issues:** Open a GitHub issue -- **Connection problems:** Check the troubleshooting section above -- **General questions:** Ask your team lead or Snowflake admin +Open a GitHub issue specifying what you need and business justification. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0d5b0c4..741fcdc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,11 +2,26 @@ name: CI on: pull_request: - branches: [main, v0.2] + branches: [main, v0.2, v0.3] push: - branches: [main, v0.2] + branches: [main, v0.2, v0.3] jobs: + secrets-scan: + name: Secrets Scanning + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: TruffleHog secrets scan + uses: trufflesecurity/trufflehog@main + with: + extra_args: --only-verified + lint: name: Lint & Format Check runs-on: ubuntu-latest @@ -49,6 +64,9 @@ jobs: - name: Install dependencies run: uv sync --all-extras --dev + - name: Validate YAML configs + run: uv run validate-config + - name: Run tests run: uv run pytest -v --tb=short env: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 722faec..9898b01 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -14,6 +14,19 @@ repos: - id: check-merge-conflict - id: mixed-line-ending + - repo: https://github.com/Yelp/detect-secrets + rev: v1.5.0 + hooks: + - id: detect-secrets + args: ['--baseline', '.secrets.baseline'] + exclude: > + (?x)^( + .*\.lock| + .*\.yaml\.SAFE| + tests/.*| + docs/.* + )$ + - repo: local hooks: - id: validate-schema-grants @@ -24,6 +37,14 @@ repos: pass_filenames: false always_run: false + - id: validate-config + name: Validate SnowDDL YAML Configs + entry: uv run validate-config --quiet + language: system + files: ^snowddl/.*\.yaml$ + pass_filenames: false + always_run: false + - repo: https://github.com/psf/black rev: 24.4.2 # Updated from 23.12.1 - stable version compatible with Python 3.12.5 hooks: diff --git a/.secrets.baseline b/.secrets.baseline new file mode 100644 index 0000000..14bbc6e --- /dev/null +++ b/.secrets.baseline @@ -0,0 +1,233 @@ +{ + "version": "1.5.0", + "plugins_used": [ + { + "name": "ArtifactoryDetector" + }, + { + "name": "AWSKeyDetector" + }, + { + "name": "AzureStorageKeyDetector" + }, + { + "name": "Base64HighEntropyString", + "limit": 4.5 + }, + { + "name": "BasicAuthDetector" + }, + { + "name": "CloudantDetector" + }, + { + "name": "DiscordBotTokenDetector" + }, + { + "name": "GitHubTokenDetector" + }, + { + "name": "GitLabTokenDetector" + }, + { + "name": "HexHighEntropyString", + "limit": 3.0 + }, + { + "name": "IbmCloudIamDetector" + }, + { + "name": "IbmCosHmacDetector" + }, + { + "name": "IPPublicDetector" + }, + { + "name": "JwtTokenDetector" + }, + { + "name": "KeywordDetector", + "keyword_exclude": "" + }, + { + "name": "MailchimpDetector" + }, + { + "name": "NpmDetector" + }, + { + "name": "OpenAIDetector" + }, + { + "name": "PrivateKeyDetector" + }, + { + "name": "PypiTokenDetector" + }, + { + "name": "SendGridDetector" + }, + { + "name": "SlackDetector" + }, + { + "name": "SoftlayerDetector" + }, + { + "name": "SquareOAuthDetector" + }, + { + "name": "StripeDetector" + }, + { + "name": "TelegramBotTokenDetector" + }, + { + "name": "TwilioKeyDetector" + } + ], + "filters_used": [ + { + "path": "detect_secrets.filters.allowlist.is_line_allowlisted" + }, + { + "path": "detect_secrets.filters.common.is_ignored_due_to_verification_policies", + "min_level": 2 + }, + { + "path": "detect_secrets.filters.heuristic.is_indirect_reference" + }, + { + "path": "detect_secrets.filters.heuristic.is_likely_id_string" + }, + { + "path": "detect_secrets.filters.heuristic.is_lock_file" + }, + { + "path": "detect_secrets.filters.heuristic.is_not_alphanumeric_string" + }, + { + "path": "detect_secrets.filters.heuristic.is_potential_uuid" + }, + { + "path": "detect_secrets.filters.heuristic.is_prefixed_with_dollar_sign" + }, + { + "path": "detect_secrets.filters.heuristic.is_sequential_string" + }, + { + "path": "detect_secrets.filters.heuristic.is_swagger_file" + }, + { + "path": "detect_secrets.filters.heuristic.is_templated_secret" + }, + { + "path": "detect_secrets.filters.regex.should_exclude_file", + "pattern": [ + "(\\.lock|\\.yaml\\.SAFE|tests/.*|docs/.*)" + ] + } + ], + "results": { + ".github/workflows/ci.yml": [ + { + "type": "Secret Keyword", + "filename": ".github/workflows/ci.yml", + "hashed_secret": "9fb7fe1217aed442b04c0f5e43b5d5a7d3287097", + "is_verified": false, + "line_number": 57 + } + ], + ".github/workflows/release.yml": [ + { + "type": "Secret Keyword", + "filename": ".github/workflows/release.yml", + "hashed_secret": "9fb7fe1217aed442b04c0f5e43b5d5a7d3287097", + "is_verified": false, + "line_number": 36 + } + ], + "README.md": [ + { + "type": "Secret Keyword", + "filename": "README.md", + "hashed_secret": "7165f6d407dc2fd68528da63260a913e71623e86", + "is_verified": false, + "line_number": 174 + } + ], + "pyproject.toml": [ + { + "type": "Secret Keyword", + "filename": "pyproject.toml", + "hashed_secret": "932da1e6368bff95652376e245fdd068dec8024f", + "is_verified": false, + "line_number": 121 + } + ], + "src/automation/README.md": [ + { + "type": "Secret Keyword", + "filename": "src/automation/README.md", + "hashed_secret": "373ee161fc7eb63bc82d686e6a855b2417f4cdbc", + "is_verified": false, + "line_number": 300 + } + ], + "src/snowddl_core/safety/agent_implementation_plan.py": [ + { + "type": "Secret Keyword", + "filename": "src/snowddl_core/safety/agent_implementation_plan.py", + "hashed_secret": "638bac731294171648258260ff2af4a09bc02aa2", + "is_verified": false, + "line_number": 41 + } + ], + "src/snowddl_core/snowddl_types.py": [ + { + "type": "Secret Keyword", + "filename": "src/snowddl_core/snowddl_types.py", + "hashed_secret": "644a049372cae070675af379d343ec1267880fe7", + "is_verified": false, + "line_number": 64 + } + ], + "src/snowtower_core/audit.py": [ + { + "type": "Secret Keyword", + "filename": "src/snowtower_core/audit.py", + "hashed_secret": "94732424e16be827cd46aa294394ab3ce93b55f4", + "is_verified": false, + "line_number": 63 + } + ], + "src/snowtower_core/managers.py": [ + { + "type": "Secret Keyword", + "filename": "src/snowtower_core/managers.py", + "hashed_secret": "5baa61e4c9b93f3f0682250b6cf8331b7ee68fd8", + "is_verified": false, + "line_number": 164 + } + ], + "src/snowtower_core/models.py": [ + { + "type": "Secret Keyword", + "filename": "src/snowtower_core/models.py", + "hashed_secret": "112bb791304791ddcf692e29fd5cf149b35fea37", + "is_verified": false, + "line_number": 36 + } + ], + "src/user_management/health_check.py": [ + { + "type": "Secret Keyword", + "filename": "src/user_management/health_check.py", + "hashed_secret": "5baa61e4c9b93f3f0682250b6cf8331b7ee68fd8", + "is_verified": false, + "line_number": 42 + } + ] + }, + "generated_at": "2026-02-21T17:15:41Z" +} diff --git a/CHANGELOG.md b/CHANGELOG.md index c9c8fb5..409f017 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,389 +1,63 @@ # Changelog -All notable changes to the SnowTower SnowDDL project will be documented in this file. +All notable changes to SnowTower are documented in this file. +Format based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). +## [0.3.0] - 2026-02-22 -## [2025-12-03] - Infrastructure Update +### Added +- `validate-config` command: Pre-deployment YAML validation with cross-reference integrity checks +- `generate-terraform` command: Convert SnowDDL YAML to Terraform HCL with import blocks +- Integration test suite: 58 tests covering CLI, authentication, YAML validation, deployment workflows +- Secrets scanning: detect-secrets pre-commit hook + TruffleHog CI job +- `validate-config` pre-commit hook for automatic YAML validation on commit +- `validate-config` CI step for validation on every push/PR +- Terraform boilerplate example output in `terraform-snowflake-boilerplate/` +- Unit tests for `validate_config.py` and `generate_terraform.py` ### Changed -- Merge pull request #75 from Database-Tycoon/fix/dlt-table-stage-permissions -- fix: Add schema_owner to DBT business role for PROJ_STRIPE access -- docs: update changelog for deployment 79410355 -- Merge pull request #74 from Database-Tycoon/fix/dlt-table-stage-permissions -- chore: Remove plan generation from PR validation workflow -- fix: Change DEV_CAROL to non-sandbox to match existing PERMANENT schema -- docs: update changelog for deployment 7559b03e -- Merge pull request #73 from Database-Tycoon/fix/dlt-table-stage-permissions -- fix: Add schema_owner to DLT business role for table stage access -- docs: Add note about DLT schema owner role requirement -- docs: update changelog for deployment 23fe7d97 -- Merge pull request #72 from Database-Tycoon/fix/ci-workflow-exit-codes -- fix: Remove obsolete transfer_ownership_to_dbt step from CI -- fix: Use uv run for transfer_ownership_to_dbt.py -- fix: Handle snowddl-apply exit code 8 as success -- fix: Handle snowddl-plan exit codes in all deployment steps - -### Infrastructure Changes -- fix: Add schema_owner to DBT business role for PROJ_STRIPE access (a542757) -- fix: Change DEV_CAROL to non-sandbox to match existing PERMANENT schema (d29f9ff) -- fix: Add schema_owner to DLT business role for table stage access (da7de9f) -- docs: Add note about DLT schema owner role requirement (dc81bf1) - -### Deployment Info -- **Commit:** 50a30bf6 -- **Timestamp:** 2025-12-03 12:45:03 UTC -- **Triggered by:** example-user - - - -## [2025-12-03] - Infrastructure Update - -### Changed -- Merge pull request #74 from Database-Tycoon/fix/dlt-table-stage-permissions -- chore: Remove plan generation from PR validation workflow -- fix: Change DEV_CAROL to non-sandbox to match existing PERMANENT schema -- docs: update changelog for deployment 7559b03e -- Merge pull request #73 from Database-Tycoon/fix/dlt-table-stage-permissions -- fix: Add schema_owner to DLT business role for table stage access -- docs: Add note about DLT schema owner role requirement -- docs: update changelog for deployment 23fe7d97 -- Merge pull request #72 from Database-Tycoon/fix/ci-workflow-exit-codes -- fix: Remove obsolete transfer_ownership_to_dbt step from CI -- fix: Use uv run for transfer_ownership_to_dbt.py -- fix: Handle snowddl-apply exit code 8 as success -- fix: Handle snowddl-plan exit codes in all deployment steps -- fix: Handle snowddl-plan non-zero exit codes in deployment workflow -- docs: update changelog for deployment 73736954 - -### Infrastructure Changes -- fix: Change DEV_CAROL to non-sandbox to match existing PERMANENT schema (d29f9ff) -- fix: Add schema_owner to DLT business role for table stage access (da7de9f) -- docs: Add note about DLT schema owner role requirement (dc81bf1) - -### Deployment Info -- **Commit:** 79410355 -- **Timestamp:** 2025-12-03 12:14:32 UTC -- **Triggered by:** example-user - - - -## [2025-12-03] - Infrastructure Update - -### Changed -- Merge pull request #73 from Database-Tycoon/fix/dlt-table-stage-permissions -- fix: Add schema_owner to DLT business role for table stage access -- docs: Add note about DLT schema owner role requirement -- docs: update changelog for deployment 23fe7d97 -- Merge pull request #72 from Database-Tycoon/fix/ci-workflow-exit-codes -- fix: Remove obsolete transfer_ownership_to_dbt step from CI -- fix: Use uv run for transfer_ownership_to_dbt.py -- fix: Handle snowddl-apply exit code 8 as success -- fix: Handle snowddl-plan exit codes in all deployment steps -- fix: Handle snowddl-plan non-zero exit codes in deployment workflow -- docs: update changelog for deployment 73736954 -- Merge pull request #71 from Database-Tycoon/feature/eliminate-schema-drift -- feat: Enable SnowDDL schema management and fix directory structure -- docs: Add SnowDDL knowledge base and fix test script -- fix: Resolve all test failures and add release checklist -- chore: Prepare for v0.1 release -- chore: Add keys/ to .gitignore and cleanup local artifacts -- fix: Correct docs path in help_cli.py -- feat: Remove web/Streamlit features for v0.1 release -- docs: restructure documentation and add LLM agent configuration - -### Infrastructure Changes -- fix: Add schema_owner to DLT business role for table stage access (da7de9f) -- docs: Add note about DLT schema owner role requirement (dc81bf1) -- feat: Enable SnowDDL schema management and fix directory structure (8d49e44) -- feat: Add schema.yaml files for all databases to eliminate schema drift (8fc2140) - -### Deployment Info -- **Commit:** 7559b03e -- **Timestamp:** 2025-12-03 11:46:16 UTC -- **Triggered by:** example-user - - - -## [2025-11-30] - Infrastructure Update - -### Changed -- Merge pull request #72 from Database-Tycoon/fix/ci-workflow-exit-codes -- fix: Remove obsolete transfer_ownership_to_dbt step from CI -- fix: Use uv run for transfer_ownership_to_dbt.py -- fix: Handle snowddl-apply exit code 8 as success -- fix: Handle snowddl-plan exit codes in all deployment steps -- fix: Handle snowddl-plan non-zero exit codes in deployment workflow -- docs: update changelog for deployment 73736954 -- Merge pull request #71 from Database-Tycoon/feature/eliminate-schema-drift -- feat: Enable SnowDDL schema management and fix directory structure -- docs: Add SnowDDL knowledge base and fix test script -- fix: Resolve all test failures and add release checklist -- chore: Prepare for v0.1 release -- chore: Add keys/ to .gitignore and cleanup local artifacts -- fix: Correct docs path in help_cli.py -- feat: Remove web/Streamlit features for v0.1 release -- docs: restructure documentation and add LLM agent configuration -- docs: Add comprehensive next steps guide for schema drift elimination -- feat: Add schema.yaml files for all databases to eliminate schema drift -- docs: Add comprehensive CHANGELOG.md for 0.1 release -- fix: Remove 6 non-working command references - -### Infrastructure Changes -- feat: Enable SnowDDL schema management and fix directory structure (8d49e44) -- feat: Add schema.yaml files for all databases to eliminate schema drift (8fc2140) -- feat: Add IP 192.0.2.10 to bi_tool_network_policy (a0d86c5) - -### Deployment Info -- **Commit:** 23fe7d97 -- **Timestamp:** 2025-11-30 23:33:51 UTC -- **Triggered by:** example-user - - - -## [2025-11-30] - Infrastructure Update +- Consolidated `.claude` resources: ~2,900 lines across 13 files reduced to ~580 lines across 5 files +- Rewrote 4 Claude Code skills as concise task guides (~60-116 lines each) +- Enhanced root `CLAUDE.md` as single source of truth with SnowDDL knowledge base +- Version bumped to 0.3.0 + +### Fixed +- Mismatched `default_role` references in `user.yaml` (ANALYST__B_ROLE, DLT_INGESTION_ROLE__B_ROLE) +- Stale references to nonexistent `.claude/agents/` directory +- Dead links to deleted `docs/llm-context/` directory + +### Removed +- `docs/llm-context/` directory (4 files) - consolidated into root CLAUDE.md +- Redundant skill files (developer README/CHANGELOG, maintainer PROJECT_STRUCTURE, skills README) + +## [0.2.0] - 2025-10-15 + +### Added +- Claude Code skills: snowtower-user, snowtower-admin, snowtower-developer, snowtower-maintainer +- CI/CD workflows: lint, test, secrets scanning via GitHub Actions +- PR template and issue templates +- CONTRIBUTING.md with development guidelines +- Pre-commit hooks for code formatting and YAML validation +- ASCII banner command (`snowtower-banner`) ### Changed -- Merge pull request #71 from Database-Tycoon/feature/eliminate-schema-drift -- feat: Enable SnowDDL schema management and fix directory structure -- docs: Add SnowDDL knowledge base and fix test script -- fix: Resolve all test failures and add release checklist -- chore: Prepare for v0.1 release -- chore: Add keys/ to .gitignore and cleanup local artifacts -- fix: Correct docs path in help_cli.py -- feat: Remove web/Streamlit features for v0.1 release -- docs: restructure documentation and add LLM agent configuration -- docs: Add comprehensive next steps guide for schema drift elimination -- feat: Add schema.yaml files for all databases to eliminate schema drift -- docs: Add comprehensive CHANGELOG.md for 0.1 release -- fix: Remove 6 non-working command references -- Merge pull request #59 from Database-Tycoon/fix/automatic-ownership-transfer-in-ci -- fix: add automatic ownership transfer to deployment workflow -- Merge pull request #58 from Database-Tycoon/fix/lightdash-network-policy-deployment -- fix: Add ownership investigation and transfer tools -- fix: Prevent dbt permission loss with deploy-safe wrapper -- feat: Add IP 192.0.2.10 to bi_tool_network_policy -- docs: Add Lightdash network policy deployment documentation - -### Infrastructure Changes -- feat: Enable SnowDDL schema management and fix directory structure (8d49e44) -- feat: Add schema.yaml files for all databases to eliminate schema drift (8fc2140) -- feat: Add IP 192.0.2.10 to bi_tool_network_policy (a0d86c5) -- Add RSA public key for CAROL user (3d8fcac) -- fix: Add CREATE SCHEMA privilege to DBT_STRIPE_ROLE for SOURCE_STRIPE (3a59c83) - -### Deployment Info -- **Commit:** 73736954 -- **Timestamp:** 2025-11-30 20:16:38 UTC -- **Triggered by:** example-user - - -The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), -and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - -## [0.1.0] - 2025-11-21 - -### 🎉 Initial Release - -The first official release of SnowTower - a unified Snowflake infrastructure management platform with comprehensive CLI commands and Infrastructure as Code capabilities. - -### ✨ Major Features - -#### Infrastructure as Code -- **SnowDDL Integration**: Complete declarative infrastructure management via YAML configurations -- **Safe Deployment Workflow**: `uv run deploy-safe` ensures schema grants are always applied after infrastructure changes -- **Intelligent Plan Filtering** ⭐: Automatic suppression of expected schema grant drift in PR reviews (see [#66](https://github.com/Database-Tycoon/snowtower/pull/66)) - - Eliminates hundreds of REVOKE statements from plan output - - Makes actual infrastructure changes crystal clear - - Preserves full audit trail in collapsible sections - - Integrated into CI/CD workflows - -#### User Management -- **Complete User Lifecycle**: Interactive and non-interactive user creation with `uv run manage-users` -- **Dual Authentication**: RSA key-pair (primary) + encrypted password (fallback) for all users -- **MFA Compliance Tracking**: Ready for Snowflake's 2025-2026 mandatory MFA rollout -- **Bulk Operations**: Password generation, CSV import, key rotation - -#### Resource Management -- **Warehouse Management**: Resize, auto-suspend, cost optimization with `uv run manage-warehouses` -- **Cost Optimization**: Analysis and recommendations with `uv run manage-costs` -- **Security Auditing**: Comprehensive security checks with `uv run manage-security` -- **Backup & Restore**: Configuration snapshots with `uv run manage-backup` - -#### Monitoring & Observability -- **Health Checks**: System health monitoring with `uv run monitor-health` -- **Audit Logs**: Complete audit trail with `uv run monitor-audit` -- **Operational Metrics**: Performance and usage metrics with `uv run monitor-metrics` - -#### CI/CD Automation -- **GitHub Actions Workflows**: Automated PR validation and production deployment -- **Safety Gates**: Pre-deployment validation, security scanning, health checks -- **Schema Grant Protection**: Automatic application of schema grants after every deployment -- **Emergency Rollback**: Snapshot-based rollback capabilities - -### 🔐 Security - -- **MFA Enforcement**: Timeline-ready for Snowflake's mandatory MFA (March 2026) -- **RSA Key Authentication**: Prioritized for all service accounts -- **Encrypted Passwords**: Fernet encryption for all password storage -- **Network Policies**: IP restrictions for human users (192.0.2.10/32) -- **Emergency Access**: STEPHEN_RECOVERY account preserved without network restrictions -- **Audit Trail**: Complete logging of all infrastructure changes - -### 📚 Documentation - -- **Comprehensive README**: User-focused onboarding and administrator guide -- **Management Commands Reference**: Complete CLI command documentation -- **Quick Start Guide**: 5-minute setup for new users -- **Schema Grants Workaround**: Detailed explanation of SnowDDL's SCHEMA exclusion pattern -- **Security Guides**: Authentication setup, MFA compliance, RSA key generation -- **0.1 Release Review**: Comprehensive pre-release audit and recommendations - -### 🛠️ CLI Commands - -**25+ Working Commands** organized in 7 categories: - -#### Core SnowDDL Operations (5 commands) -- `snowddl-plan` - Preview infrastructure changes with intelligent filtering -- `snowddl-apply` - Apply infrastructure changes -- `snowddl-validate` - Validate YAML configurations -- `snowddl-diff` - Show differences -- `deploy-safe` - Safe deployment with automatic schema grants ⭐ - -#### User Management (1 command with subcommands) -- `manage-users` - Complete user lifecycle (create, list, update, delete, validate, etc.) - -#### Resource Management (6 commands) -- `manage-warehouses` - Warehouse operations and optimization -- `manage-costs` - Cost analysis and optimization -- `manage-security` - Security auditing -- `manage-backup` - Configuration backup/restore -- `apply-schema-grants` - Apply schema-level grants ⭐ -- `validate-schema-grants` - Validate schema grant consistency - -#### Monitoring (3 commands) -- `monitor-health` - System health checks -- `monitor-audit` - Audit trail analysis -- `monitor-metrics` - Operational metrics - -#### Web Interface (2 commands) -- `web` - Launch Streamlit dashboard -- `deploy-streamlit` - Deploy Streamlit apps to Snowflake - -#### Utilities (4 commands) -- `util-generate-key` - Generate Fernet encryption key -- `util-diagnose-auth` - Diagnose authentication issues -- `util-fix-auth` - Fix authentication problems -- `generate-rsa-batch` - Batch RSA key generation - -#### Automation (2 commands) -- `github-to-snowddl` - Convert GitHub issues to SnowDDL PRs -- `process-access-request` - Process user access requests - -#### Documentation (2 commands) -- `docs-serve` - Serve documentation locally -- `docs-build` - Build documentation - -#### New in 0.1 (1 command) -- `filter-plan` - Intelligent plan output filtering ⭐ - -### 🐛 Fixed - -- **Schema Grant Drift Noise**: Intelligent filtering eliminates hundreds of REVOKE statements from PR reviews -- **Missing Command References**: Removed 6 non-existent commands from registry - - `test-s3-deployment` - - `sync-s3-configs` - - `test-streamlit-local` - - `test-streamlit-deployed` - - `deploy-streamlit-safe` - - `detect-streamlit-errors` -- **Documentation Accuracy**: Updated command documentation to reflect actual implementations - -### ⚠️ Known Limitations - -#### Web UI -- **Status**: Has known errors and issues -- **Recommendation**: Use CLI commands instead -- **Launch Command**: `uv run web` works but may have runtime errors -- **Roadmap**: Full web UI refactoring planned for 0.2 release - -#### Test Coverage -- **Current**: ~40% (filter tests only) -- **Target**: >80% by 0.2 release -- **Status**: Integration tests planned but not yet implemented - -#### Streamlit Testing -- Basic validation available via `validate-streamlit` -- Advanced testing commands deferred to future release - -### 🚀 Deployment - -#### Prerequisites -- Python 3.10+ -- UV package manager -- Snowflake account with ACCOUNTADMIN role -- RSA key pair for authentication - -#### Quick Start -```bash -# 1. Clone and install -git clone https://github.com/Database-Tycoon/snowtower.git -cd snowtower-snowddl -uv sync - -# 2. Configure authentication -cp .env.example .env -# Edit .env with your credentials - -# 3. Preview changes -uv run snowddl-plan - -# 4. Deploy safely -uv run deploy-safe -``` - -### 📊 Project Statistics - -- **Active Users**: 13 configured (7 human, 5 service accounts) -- **Databases Managed**: 6 production databases -- **Warehouses**: 8 with auto-suspend -- **Resource Monitors**: 7 active cost monitors -- **Security Policies**: Network and authentication policies enforced -- **Test Coverage**: 13 passing tests for plan filtering - -### 🔄 Migration Guide - -This is the initial release, so no migration is required. For users of the pre-release versions: - -1. **Update Dependencies**: Run `uv sync` to install all dependencies -2. **Review Commands**: Some commands have been removed - see Fixed section above -3. **Update Workflows**: CI/CD workflows now include intelligent filtering -4. **Review Documentation**: New comprehensive guides in `docs/` directory - -### 🙏 Acknowledgments - -- Built with [SnowDDL](https://github.com/littleK0i/SnowDDL) by littleK0i -- Powered by [UV](https://github.com/astral-sh/uv) package manager by Astral -- Infrastructure automation via [GitHub Actions](https://github.com/features/actions) - -### 📝 Notes - -- **Recommended Deployment**: Always use `uv run deploy-safe` instead of `snowddl-apply` -- **Schema Grants**: See `docs/SCHEMA_GRANTS_WORKAROUND.md` for critical information -- **MFA Timeline**: Enable MFA for human users before March 2026 deadline -- **RSA Keys**: Primary authentication method for all service accounts - ---- - -## [Unreleased] - -### Planned for 0.2.0 -- Complete web UI refactoring and error fixes -- Integration test suite for CLI commands -- Increased test coverage (>80%) -- Advanced Streamlit testing commands -- API documentation generation -- Multi-account support exploration - ---- - -**Full Changelog**: https://github.com/Database-Tycoon/snowtower/commits/v0.1.0 +- Consolidated from dual-repo to single unified platform +- Updated README with badges and improved documentation + +## [0.1.0] - 2025-08-01 + +### Added +- Initial release of SnowTower enterprise Snowflake infrastructure management +- SnowDDL YAML-based infrastructure definitions (users, roles, warehouses, policies) +- User management with Fernet encryption and RSA key authentication +- OOP framework for SnowDDL operations (`snowddl_core`) +- Management CLI: warehouses, costs, security, backup, users +- Monitoring: health, audit, metrics +- Deploy-safe wrapper with safety checks +- Network, authentication, password, and session policies +- Resource monitors with trigger configuration +- Comprehensive documentation (README, QUICKSTART, RSA_KEY_SETUP) +- MFA compliance tracking for Snowflake 2025-2026 rollout +- 25+ CLI commands across 7 categories + +**Full Changelog**: https://github.com/Database-Tycoon/SnowTower/commits/main diff --git a/CLAUDE.md b/CLAUDE.md index 8f44ef0..e3b8a3d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -8,10 +8,12 @@ # Essential commands uv run snowddl-plan # Preview infrastructure changes uv run snowddl-apply # Apply changes (needs confirmation) +uv run deploy-safe # Safe deployment (preserves schema grants) uv run manage-users # User management uv run manage-warehouses # Warehouse management uv run manage-costs # Cost analysis uv sync # Install dependencies +uv run pytest # Run tests ``` ## Critical Rules @@ -41,11 +43,11 @@ snowtower-snowddl/ │ ├── snowddl_core/ # OOP framework │ └── management_cli.py # CLI entry points ├── scripts/ # Management scripts +├── tests/ # Test suite ├── docs/ # Documentation │ ├── guide/ # User guides │ ├── contributing/ # Developer docs -│ ├── llm-context/ # LLM configuration -│ └── ... +│ └── releases/ # Release docs └── pyproject.toml # UV/Python config ``` @@ -104,6 +106,47 @@ The `--env-prefix` parameter adds a prefix to all object names (databases, schem **Must use `-r ACCOUNTADMIN`** to see all grants across roles/schemas when running plan. +### SnowDDL Object Type Reference + +| Object Type | Grant Key | Example | +|------------|-----------|---------| +| Database | `DATABASE:` | `DATABASE:USAGE,CREATE SCHEMA` | +| Schema | `SCHEMA:` | `SCHEMA:USAGE,CREATE TABLE` | +| Table | `TABLE:` | `TABLE:SELECT,INSERT` | +| View | `VIEW:` | `VIEW:SELECT` | +| Warehouse | `WAREHOUSE:` | `WAREHOUSE:USAGE,OPERATE` | +| Stage | `STAGE:` | `STAGE:USAGE,READ,WRITE` | +| Function | `FUNCTION:` | `FUNCTION:USAGE` | +| Procedure | `PROCEDURE:` | `PROCEDURE:USAGE` | + +### Role Types and Naming + +SnowDDL automatically creates roles with specific suffixes: + +| Role Type | Suffix | Purpose | Example | +|-----------|--------|---------|---------| +| User Role | `__U_ROLE` | Per-user role | `DLT__U_ROLE` | +| Technical Role | `__T_ROLE` | Service/app permissions | `DLT_STRIPE_TECH_ROLE__T_ROLE` | +| Business Role | `__B_ROLE` | Logical groupings | `DLT_STRIPE_ROLE__B_ROLE` | +| Schema Owner | `__S_ROLE` | Object ownership | `SOURCE_STRIPE__STRIPE_WHY__OWNER__S_ROLE` | +| Database Owner | `__D_ROLE` | Database-level ownership | `SOURCE_STRIPE__OWNER__D_ROLE` | + +### Role Hierarchy (RBAC) + +``` +ACCOUNTADMIN (top-level) + ↓ +SYSADMIN → USERADMIN → SECURITYADMIN + ↓ ↓ ↓ +Business Roles (__B_ROLE) + ↓ +Technical Roles (__T_ROLE) + ↓ +Object Permissions +``` + +**Inheritance**: User → User Role (__U_ROLE) → Business Role (__B_ROLE) → Technical Role (__T_ROLE) + Schema Owner (__S_ROLE) + ### Schema Grants in SnowDDL Schema grants are defined in `tech_role.yaml` using `SCHEMA:` format: @@ -112,13 +155,11 @@ Schema grants are defined in `tech_role.yaml` using `SCHEMA:` format: # tech_role.yaml DBT_STRIPE_ROLE: grants: - # Schema-level grants (the key insight!) SCHEMA:USAGE: - SOURCE_STRIPE.STRIPE_WHY - PROJ_STRIPE.PROJ_STRIPE SCHEMA:CREATE TABLE,CREATE VIEW,MODIFY: - PROJ_STRIPE.PROJ_STRIPE - # Database grants DATABASE:USAGE,CREATE SCHEMA: - SOURCE_STRIPE - PROJ_STRIPE @@ -157,6 +198,7 @@ The "schema drift" issue occurs when: - [Schema Config](https://docs.snowddl.com/basic/yaml-configs/schema) - Schema params.yaml structure - [Technical Roles](https://docs.snowddl.com/basic/yaml-configs/technical-role) - Grant definitions - [Business Roles](https://docs.snowddl.com/basic/yaml-configs/business-role) - schema_read/write abstractions +- [Permission Model](https://docs.snowddl.com/basic/yaml-configs/permission-model) - Permission models - [Env Prefix Guide](https://docs.snowddl.com/guides/other-guides/env-prefix) - Environment separation ## Setting Up dbt Projects (CRITICAL) @@ -203,15 +245,6 @@ GRANT OWNERSHIP ON ALL VIEWS IN SCHEMA . TO ROLE ____OWNER__S_ROLE COPY CURRENT GRANTS; ``` -**Real example for PROJ_STRIPE.PROJ_STRIPE:** -```sql -GRANT OWNERSHIP ON ALL TABLES IN SCHEMA PROJ_STRIPE.PROJ_STRIPE - TO ROLE PROJ_STRIPE__PROJ_STRIPE__OWNER__S_ROLE COPY CURRENT GRANTS; - -GRANT OWNERSHIP ON ALL VIEWS IN SCHEMA PROJ_STRIPE.PROJ_STRIPE - TO ROLE PROJ_STRIPE__PROJ_STRIPE__OWNER__S_ROLE COPY CURRENT GRANTS; -``` - ### Why This Works The grant chain after setup: @@ -232,9 +265,49 @@ SQL access control error: View 'MY_VIEW' already exists, but current role has no **Fix:** Complete all 3 steps above. +## DLT (Data Loading) Permission Patterns + +### The Table Stage Problem + +When DLT loads data, it uses `COPY INTO` with table stages. **Only the TABLE OWNER can access table stages.** If DLT doesn't own the tables, loading fails with: + +``` +SQL access control error: Insufficient privileges to operate on table stage 'CHARGE' +``` + +### Solution: Grant Schema Owner Role to Business Role + +```yaml +# snowddl/business_role.yaml +DLT_STRIPE_ROLE: + comment: DLT Stripe pipeline - includes schema owner for table stage access + tech_roles: + - DLT_STRIPE_TECH_ROLE + schema_owner: + - SOURCE_STRIPE.STRIPE_WHY # Grants SOURCE_STRIPE__STRIPE_WHY__OWNER__S_ROLE + warehouse_usage: + - DLT +``` + +**Why this works**: `schema_owner` grants the auto-created `__OWNER__S_ROLE` to the business role. The user inherits through: User Role → Business Role → Schema Owner Role → table stage access. + +**What doesn't work**: Technical roles don't support `schema_owner`. You must use business roles. + +## Environment Variables + +Required in `.env`: +``` +SNOWFLAKE_ACCOUNT=your_account +SNOWFLAKE_USER=your_user +SNOWFLAKE_PASSWORD=your_password # Or use RSA key +SNOWFLAKE_ROLE=ACCOUNTADMIN +SNOWFLAKE_WAREHOUSE=COMPUTE_WH +FERNET_KEY=your_fernet_key # For password encryption +``` + ## For More Details -- [Full LLM Instructions](docs/llm-context/CLAUDE.md) -- [Project Context](docs/llm-context/CONTEXT.md) -- [Code Patterns](docs/llm-context/PATTERNS.md) - [CLI Reference](docs/guide/MANAGEMENT_COMMANDS.md) +- [Troubleshooting](docs/guide/TROUBLESHOOTING.md) +- [Architecture](docs/guide/ARCHITECTURE.md) +- [Service Account Pattern](.claude/patterns/SERVICE_ACCOUNT_CREATION_PATTERN.md) diff --git a/README.md b/README.md index 0b119ed..4c13ec5 100644 --- a/README.md +++ b/README.md @@ -629,6 +629,7 @@ This project includes Claude Code skills for AI-assisted workflows: |-------|---------|----------| | [snowtower-user](.claude/skills/snowtower-user/) | End-user guide | Requesting access, connecting to Snowflake | | [snowtower-admin](.claude/skills/snowtower-admin/) | Admin operations | SnowDDL deployments, user management, troubleshooting | +| [snowtower-developer](.claude/skills/snowtower-developer/) | Code contributions | Adding features, fixing bugs, writing tests, submitting PRs | | [snowtower-maintainer](.claude/skills/snowtower-maintainer/) | Project maintenance | README updates, documentation sync | ### Full Documentation diff --git a/docs/README.md b/docs/README.md index 37eff3a..23ea5a3 100644 --- a/docs/README.md +++ b/docs/README.md @@ -10,8 +10,6 @@ | Fix schema permissions | [Schema Grants](guide/SCHEMA_GRANTS.md) | | Troubleshoot issues | [Troubleshooting](guide/TROUBLESHOOTING.md) | | View the changelog | [Changelog](releases/CHANGELOG.md) | -| Use LLMs with this repo | [LLM Context](llm-context/) | - --- ## Guide @@ -41,17 +39,6 @@ All user-facing documentation. --- -## LLM Context - -Configuration files for using LLMs with this codebase. - -- [README](llm-context/README.md) - Overview and quick start -- [CLAUDE.md](llm-context/CLAUDE.md) - Full project instructions for Claude -- [CONTEXT.md](llm-context/CONTEXT.md) - Domain knowledge and project context -- [PATTERNS.md](llm-context/PATTERNS.md) - Code patterns and conventions - ---- - ## Contributing For developers and contributors. diff --git a/docs/llm-context/CLAUDE.md b/docs/llm-context/CLAUDE.md deleted file mode 100644 index 42af253..0000000 --- a/docs/llm-context/CLAUDE.md +++ /dev/null @@ -1,329 +0,0 @@ -# CLAUDE.md - SnowTower SnowDDL Project Instructions - -## Project Overview - -**SnowTower SnowDDL** is an enterprise Snowflake infrastructure management platform using Infrastructure as Code (IaC) with SnowDDL. - -### What This Project Does -- Manages Snowflake infrastructure via declarative YAML files -- Provides CLI commands for user, warehouse, and cost management -- Includes security policies, network policies, and MFA compliance - -## Critical Rules - -### Always Do -1. **Use UV for all Python operations**: `uv run `, `uv sync`, `uv add ` -2. **Load environment variables**: Every script must start with `load_dotenv()` -3. **Run plan before apply**: Always `uv run snowddl-plan` before `uv run snowddl-apply` -4. **Follow existing patterns**: Check `scripts/` for examples before creating new commands - -### Never Do -1. **Never run `snowddl-apply` without user confirmation** -2. **Never commit credentials or secrets** -3. **Never modify `snowddl/*.yaml` without understanding the impact** -4. **Never use pip directly** - always use UV - -## Key Commands - -```bash -# Infrastructure -uv run snowddl-plan # Preview changes (ALWAYS RUN FIRST) -uv run snowddl-apply # Apply changes (REQUIRES CONFIRMATION) -uv run deploy-safe # Recommended: deploys + applies schema grants - -# User Management -uv run manage-users # Full user management CLI - -# Operations -uv run manage-warehouses # Warehouse management -uv run manage-costs # Cost analysis -uv run monitor-health # System health - -# Development -uv sync # Install dependencies -uv run pytest # Run tests -``` - -## Project Structure - -``` -snowtower-snowddl/ -├── snowddl/ # Infrastructure definitions (YAML) -│ ├── user.yaml # User definitions -│ ├── role.yaml # Role definitions -│ ├── warehouse.yaml # Warehouse configs -│ ├── network_policy.yaml # Network policies -│ └── {DATABASE}/ # Database-specific configs -├── src/ -│ ├── snowddl_core/ # OOP framework -│ ├── user_management/ # User lifecycle -│ └── management_cli.py # CLI entry points -├── scripts/ # Management scripts -├── docs/ # Documentation -└── pyproject.toml # UV/Python config -``` - -## Creating New Commands - -Follow this pattern: - -### 1. Create Script in `scripts/` - -```python -#!/usr/bin/env python3 -"""Description of what this script does.""" - -from dotenv import load_dotenv -load_dotenv() # MANDATORY - must be first - -import argparse -from src.snowddl_core import SnowDDLClient - -def main(): - parser = argparse.ArgumentParser(description="Command description") - parser.add_argument("--option", help="Option description") - args = parser.parse_args() - - # Implementation here - -if __name__ == "__main__": - main() -``` - -### 2. Add Wrapper in `src/management_cli.py` - -```python -def my_command(): - """Run my command.""" - from scripts.my_script import main - main() -``` - -### 3. Register in `pyproject.toml` - -```toml -[project.scripts] -my-command = "src.management_cli:my_command" -``` - -### 4. Test - -```bash -uv sync -uv run my-command --help -``` - -## YAML Configuration Patterns - -### User Definition -```yaml -# snowddl/user.yaml -ANALYST_USER: - type: PERSON - default_role: ANALYST_ROLE - default_warehouse: ANALYST_WH - must_change_password: false - network_policy: OFFICE_NETWORK_POLICY -``` - -### Role Definition -```yaml -# snowddl/role.yaml -ANALYST_ROLE: - comment: "Read-only analyst access" - grants: - - database: ANALYTICS - privileges: [USAGE] -``` - -### Service Account (RSA Key Auth) -```yaml -# snowddl/user.yaml -DBT_SERVICE: - type: SERVICE - default_role: DBT_ROLE - rsa_public_key: | - -----BEGIN PUBLIC KEY----- - MIIBIjANBgk... - -----END PUBLIC KEY----- -``` - -## Security Considerations - -### Authentication Hierarchy -1. **RSA Key Pairs** - Preferred for all users -2. **Encrypted Passwords** - Fallback only (Fernet encryption) -3. **MFA** - Required for human users by 2026 - -### Network Policies -- Human users: Restricted to office IP -- Service accounts: Typically unrestricted -- Emergency access: STEPHEN_RECOVERY has no network policy - -## Common Tasks - -### Add a New User -```bash -uv run manage-users create # Interactive wizard -uv run snowddl-plan # Preview -uv run deploy-safe # Apply (with schema grants) -``` - -### Check System Health -```bash -uv run monitor-health -uv run manage-costs --analyze -``` - -### Debug SnowDDL Issues -```bash -uv run snowddl-plan --verbose -# Check snowddl/*.yaml for syntax errors -``` - -## Environment Variables - -Required in `.env`: -``` -SNOWFLAKE_ACCOUNT=your_account -SNOWFLAKE_USER=your_user -SNOWFLAKE_PASSWORD=your_password # Or use RSA key -SNOWFLAKE_ROLE=ACCOUNTADMIN -SNOWFLAKE_WAREHOUSE=COMPUTE_WH -FERNET_KEY=your_fernet_key # For password encryption -``` - -## SnowDDL Deep Knowledge - -### Critical: Understanding `--env-prefix` - -**The `--env-prefix` parameter is NOT for environment variable naming!** - -```bash -# WRONG understanding - This does NOT read from SNOWFLAKE_* env vars -snowddl --env-prefix SNOWFLAKE plan - -# What it ACTUALLY does - Prefixes all OBJECT NAMES -# It looks for objects like: SNOWFLAKE__MY_DATABASE, SNOWFLAKE__MY_ROLE, etc. -``` - -The `--env-prefix` adds a prefix to object names for environment separation (DEV__, PROD__, etc.). -SnowDDL reads connection credentials from environment variables automatically (SNOWFLAKE_ACCOUNT, SNOWFLAKE_USER, etc.) without needing `--env-prefix`. - -**Correct usage**: -```bash -# Use -r ACCOUNTADMIN to see all grants when planning -snowddl -c snowddl -r ACCOUNTADMIN plan - -# For environment separation (multiple envs on same account) -snowddl -c snowddl --env-prefix DEV plan # Creates DEV__DATABASE, DEV__ROLE, etc. -``` - -### Schema Grants in Technical Roles - -Schema grants must be defined in `tech_role.yaml` using `SCHEMA:` format: - -```yaml -# snowddl/tech_role.yaml -DBT_STRIPE_ROLE: - comment: dbt service role for Stripe transformations - grants: - # SCHEMA grants - the key to eliminating drift! - SCHEMA:USAGE: - - SOURCE_STRIPE.STRIPE_WHY - - PROJ_STRIPE.PROJ_STRIPE - SCHEMA:CREATE TABLE,CREATE VIEW,CREATE PROCEDURE,MODIFY: - - PROJ_STRIPE.PROJ_STRIPE - # DATABASE grants - DATABASE:USAGE,CREATE SCHEMA: - - SOURCE_STRIPE - - PROJ_STRIPE - # WAREHOUSE grants - WAREHOUSE:USAGE: - - TRANSFORMING - future_grants: - TABLE:SELECT: - - SOURCE_STRIPE - TABLE:SELECT,INSERT,UPDATE,DELETE,TRUNCATE: - - PROJ_STRIPE -``` - -**Grant format**: `::` followed by list of fully-qualified object names. - -### Understanding Plan Drift (REVOKE Statements) - -When `uv run snowddl-plan` shows many REVOKE statements: - -``` -REVOKE USAGE ON SCHEMA "SOURCE_STRIPE"."STRIPE_WHY" FROM ROLE "BI_WRITER_TECH_ROLE__T_ROLE"; -REVOKE ALL ON SCHEMA "PROJ_STRIPE"."PROJ_STRIPE" FROM ROLE "DBT_STRIPE_ROLE__T_ROLE"; -``` - -This means: -1. Grants exist in Snowflake that aren't defined in YAML -2. SnowDDL wants to revoke them to match the declared state -3. These often come from dbt or manual GRANT commands - -**Common drift sources**: -- dbt runs that create schemas and grant permissions -- Manual `GRANT` commands run by admins -- Other tools (Lightdash, Omni) that manage their own permissions - -**Solution**: Add the missing `SCHEMA:` grants to `tech_role.yaml`. - -### Eliminating Schema Drift - -**Problem**: Every plan shows 100s of REVOKE statements for schema grants. - -**Root cause**: dbt and other tools grant schema permissions that aren't tracked in SnowDDL. - -**Solution steps**: -1. Run `uv run snowddl-plan` and capture output -2. Parse REVOKE statements to identify: - - Which roles need grants - - Which schemas they need access to - - What privileges they have -3. Add `SCHEMA:` entries to appropriate roles in `tech_role.yaml` -4. Re-run plan to verify drift is eliminated - -**Example parsing**: -``` -REVOKE USAGE ON SCHEMA "SOURCE_STRIPE"."STRIPE_WHY" FROM ROLE "BI_WRITER_TECH_ROLE__T_ROLE" - ↓ ↓ ↓ - Privilege Schema name Role name (without __T_ROLE suffix) -``` - -Add to `tech_role.yaml`: -```yaml -BI_WRITER_TECH_ROLE: - grants: - SCHEMA:USAGE: - - SOURCE_STRIPE.STRIPE_WHY -``` - -### SnowDDL Object Type Reference - -| Object Type | Grant Key | Example | -|------------|-----------|---------| -| Database | `DATABASE:` | `DATABASE:USAGE,CREATE SCHEMA` | -| Schema | `SCHEMA:` | `SCHEMA:USAGE,CREATE TABLE` | -| Table | `TABLE:` | `TABLE:SELECT,INSERT` | -| View | `VIEW:` | `VIEW:SELECT` | -| Warehouse | `WAREHOUSE:` | `WAREHOUSE:USAGE,OPERATE` | -| Stage | `STAGE:` | `STAGE:USAGE,READ,WRITE` | -| Function | `FUNCTION:` | `FUNCTION:USAGE` | -| Procedure | `PROCEDURE:` | `PROCEDURE:USAGE` | - -### SnowDDL Documentation Links - -- [Schema Config](https://docs.snowddl.com/basic/yaml-configs/schema) -- [Technical Roles](https://docs.snowddl.com/basic/yaml-configs/technical-role) -- [Business Roles](https://docs.snowddl.com/basic/yaml-configs/business-role) -- [Permission Model](https://docs.snowddl.com/basic/yaml-configs/permission-model) -- [Env Prefix Guide](https://docs.snowddl.com/guides/other-guides/env-prefix) - -## Getting Help - -- **Troubleshooting**: `docs/guide/TROUBLESHOOTING.md` -- **CLI Reference**: `docs/guide/MANAGEMENT_COMMANDS.md` -- **Architecture**: `docs/guide/ARCHITECTURE.md` diff --git a/docs/llm-context/CONTEXT.md b/docs/llm-context/CONTEXT.md deleted file mode 100644 index 53d6140..0000000 --- a/docs/llm-context/CONTEXT.md +++ /dev/null @@ -1,90 +0,0 @@ -# Project Context - -## Domain Knowledge - -### What is SnowDDL? -SnowDDL is an open-source tool for managing Snowflake infrastructure as code. It: -- Reads YAML configuration files -- Compares desired state with actual Snowflake state -- Generates and executes DDL statements to reconcile differences - -### What is SnowTower? -SnowTower is our enterprise wrapper around SnowDDL that adds: -- User lifecycle management with MFA compliance -- Cost optimization and monitoring -- Security policies and network controls -- CLI commands via UV package manager - -## Key Concepts - -### Infrastructure as Code (IaC) -All Snowflake objects are defined in YAML files under `snowddl/`: -- Changes are version-controlled in git -- Deployments go through CI/CD (GitHub Actions) -- `plan` shows changes, `apply` executes them - -### Role-Based Access Control (RBAC) -``` -ACCOUNTADMIN (top-level) - ↓ -SYSADMIN → USERADMIN → SECURITYADMIN - ↓ ↓ ↓ -Business Roles (__B_ROLE) - ↓ -Technical Roles (__T_ROLE) - ↓ -Object Permissions -``` - -### User Types -- **PERSON**: Human users, require MFA, have network policies -- **SERVICE**: Service accounts, use RSA keys, typically unrestricted - -### Authentication Methods -1. **RSA Key Pairs** - Passwordless, most secure -2. **Encrypted Passwords** - Fernet encryption, fallback only -3. **MFA** - Multi-factor auth, mandatory for humans by 2026 - -## Technology Stack - -| Component | Technology | -|-----------|------------| -| Infrastructure | Snowflake (cloud data warehouse) | -| IaC Tool | SnowDDL | -| Language | Python 3.10+ | -| Package Manager | UV (fast pip alternative) | -| CI/CD | GitHub Actions | -| Encryption | Fernet (symmetric) | - -## Snowflake-Specific Knowledge - -### Object Hierarchy -``` -Account - └── Database - └── Schema - ├── Tables - ├── Views - ├── Stages - └── Procedures -``` - -### Common Privileges -- `USAGE` - Access to use an object -- `SELECT` - Read data -- `INSERT/UPDATE/DELETE` - Modify data -- `CREATE TABLE/VIEW` - Create objects -- `OWNERSHIP` - Full control - -### SnowDDL Limitations -- Does not manage SCHEMA objects (conflicts with dbt) -- Schema grants must be handled separately -- Some object types require `--apply-unsafe` flag - -## Project History - -- **2024**: Initial SnowDDL implementation -- **2025 Q1**: Security hardening (MFA, network policies) -- **2025 Q2**: CLI consolidation, UV migration -- **2025 Q3**: Repository unification (snowtower-cli merged) -- **Current**: Documentation restructure, agent support diff --git a/docs/llm-context/PATTERNS.md b/docs/llm-context/PATTERNS.md deleted file mode 100644 index d260219..0000000 --- a/docs/llm-context/PATTERNS.md +++ /dev/null @@ -1,491 +0,0 @@ -# Code Patterns and Conventions - -## Python Patterns - -### Script Template -Every script in `scripts/` should follow this pattern: - -```python -#!/usr/bin/env python3 -""" -Script description. - -Usage: - uv run command-name [options] -""" - -from dotenv import load_dotenv -load_dotenv() # ALWAYS FIRST - -import argparse -import sys -from pathlib import Path - -# Add src to path if needed -sys.path.insert(0, str(Path(__file__).parent.parent)) - -from src.snowddl_core import SnowDDLClient - - -def parse_args(): - parser = argparse.ArgumentParser( - description="Command description", - formatter_class=argparse.RawDescriptionHelpFormatter - ) - parser.add_argument( - "--verbose", "-v", - action="store_true", - help="Enable verbose output" - ) - parser.add_argument( - "--dry-run", - action="store_true", - help="Show what would be done without executing" - ) - return parser.parse_args() - - -def main(): - args = parse_args() - - try: - client = SnowDDLClient() - # Implementation here - - except Exception as e: - print(f"Error: {e}", file=sys.stderr) - sys.exit(1) - - -if __name__ == "__main__": - main() -``` - -### CLI Wrapper Pattern -In `src/management_cli.py`: - -```python -def my_command(): - """Short description for --help.""" - from scripts.my_script import main - main() -``` - -### Configuration Loading Pattern -```python -import os -from dotenv import load_dotenv - -load_dotenv() - -SNOWFLAKE_CONFIG = { - "account": os.getenv("SNOWFLAKE_ACCOUNT"), - "user": os.getenv("SNOWFLAKE_USER"), - "password": os.getenv("SNOWFLAKE_PASSWORD"), - "role": os.getenv("SNOWFLAKE_ROLE", "SYSADMIN"), - "warehouse": os.getenv("SNOWFLAKE_WAREHOUSE", "COMPUTE_WH"), -} -``` - -## YAML Patterns - -### User Definition -```yaml -# Standard human user -USERNAME: - type: PERSON - default_role: USER_ROLE - default_warehouse: USER_WH - must_change_password: false - network_policy: OFFICE_NETWORK_POLICY - comment: "Description of user purpose" -``` - -### Service Account with RSA -```yaml -# Service account (no password) -SERVICE_NAME: - type: SERVICE - default_role: SERVICE_ROLE - default_warehouse: SERVICE_WH - rsa_public_key: | - -----BEGIN PUBLIC KEY----- - MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A... - -----END PUBLIC KEY----- - comment: "Service account for X integration" -``` - -### Role with Grants -```yaml -ROLE_NAME: - comment: "Role description" - grants: - - database: DATABASE_NAME - privileges: [USAGE] - - schema: DATABASE_NAME.SCHEMA_NAME - privileges: [USAGE, SELECT] - - warehouse: WAREHOUSE_NAME - privileges: [USAGE] -``` - -### Network Policy -```yaml -POLICY_NAME: - allowed_ip_list: - - "10.0.0.0/8" - - "192.168.1.0/24" - blocked_ip_list: [] - comment: "Office network access only" -``` - -## Naming Conventions - -### Files -- Scripts: `snake_case.py` (e.g., `manage_users.py`) -- YAML configs: `snake_case.yaml` (e.g., `network_policy.yaml`) -- Documentation: `UPPER_CASE.md` (e.g., `QUICKSTART.md`) - -### Snowflake Objects -- Users: `UPPERCASE` (e.g., `ANALYST_USER`) -- Roles: `UPPERCASE_ROLE` (e.g., `ANALYST_ROLE`) -- Warehouses: `UPPERCASE_WH` (e.g., `COMPUTE_WH`) -- Databases: `UPPERCASE` (e.g., `RAW`, `ANALYTICS`) - -### Python -- Functions: `snake_case` -- Classes: `PascalCase` -- Constants: `UPPER_SNAKE_CASE` -- Private: `_leading_underscore` - -## Error Handling Pattern - -```python -import sys -from typing import Optional - - -class SnowTowerError(Exception): - """Base exception for SnowTower errors.""" - pass - - -class ConfigurationError(SnowTowerError): - """Configuration-related errors.""" - pass - - -def safe_operation(func): - """Decorator for safe operation execution.""" - def wrapper(*args, **kwargs): - try: - return func(*args, **kwargs) - except SnowTowerError as e: - print(f"Error: {e}", file=sys.stderr) - sys.exit(1) - except Exception as e: - print(f"Unexpected error: {e}", file=sys.stderr) - sys.exit(2) - return wrapper -``` - -## Testing Patterns - -### Unit Test -```python -import pytest -from scripts.my_script import my_function - - -def test_my_function_success(): - result = my_function("valid_input") - assert result == expected_output - - -def test_my_function_error(): - with pytest.raises(ValueError): - my_function("invalid_input") -``` - -### Integration Test -```python -import pytest -from unittest.mock import patch - - -@pytest.fixture -def mock_snowflake(): - with patch("src.snowddl_core.SnowDDLClient") as mock: - yield mock - - -def test_integration(mock_snowflake): - mock_snowflake.return_value.execute.return_value = [] - # Test implementation -``` - -## SnowDDL Role Hierarchy Patterns - -### Role Types and Naming - -SnowDDL automatically creates roles with specific suffixes: - -| Role Type | Suffix | Purpose | Example | -|-----------|--------|---------|---------| -| User Role | `__U_ROLE` | Per-user role | `DLT__U_ROLE` | -| Technical Role | `__T_ROLE` | Service/app permissions | `DLT_STRIPE_TECH_ROLE__T_ROLE` | -| Business Role | `__B_ROLE` | Logical groupings | `DLT_STRIPE_ROLE__B_ROLE` | -| Schema Owner | `__S_ROLE` | Object ownership | `SOURCE_STRIPE__STRIPE_WHY__OWNER__S_ROLE` | -| Database Owner | `__D_ROLE` | Database-level ownership | `SOURCE_STRIPE__OWNER__D_ROLE` | - -### Role Inheritance Hierarchy - -``` -User Role (DLT__U_ROLE) - ├── Business Role (DLT_STRIPE_ROLE__B_ROLE) - │ ├── Technical Role (DLT_STRIPE_TECH_ROLE__T_ROLE) - │ └── Schema Owner Role (SOURCE_STRIPE__STRIPE_WHY__OWNER__S_ROLE) - │ - └── Technical Role (directly granted if needed) -``` - -**Key insight**: Business roles INHERIT from technical roles AND can have schema_owner. -This means permissions flow: User → Business → Technical + Schema Owner. - -## Data Loading (DLT) Permission Patterns - -### The Table Stage Problem - -When DLT loads data, it uses Snowflake's `COPY INTO` with table stages: - -```sql -COPY INTO my_table FROM @my_table -- @table_name is the table stage -``` - -**Critical requirement**: Only the TABLE OWNER can access table stages. - -If DLT doesn't own the tables (common with SnowDDL's default SCHEMA_OWNER permission model), -loading fails with: - -``` -SQL access control error: Insufficient privileges to operate on table stage 'CHARGE' -``` - -### Solution: Grant Schema Owner Role to Business Role - -SnowDDL's `schema_owner` property on business roles is the correct solution: - -```yaml -# snowddl/business_role.yaml -DLT_STRIPE_ROLE: - comment: DLT Stripe pipeline - includes schema owner for table stage access - tech_roles: - - DLT_STRIPE_TECH_ROLE - schema_owner: - - SOURCE_STRIPE.STRIPE_WHY # Grants SOURCE_STRIPE__STRIPE_WHY__OWNER__S_ROLE - warehouse_usage: - - DLT -``` - -**Why this works**: -1. `schema_owner` grants the auto-created `SOURCE_STRIPE__STRIPE_WHY__OWNER__S_ROLE` to the business role -2. The user's role inherits from the business role (DLT__U_ROLE → DLT_STRIPE_ROLE__B_ROLE) -3. Therefore, the DLT user can access table stages through this inheritance chain - -### What Doesn't Work (and Why) - -**Technical roles don't support `schema_owner`:** -```yaml -# tech_role.yaml - This WON'T work! -DLT_STRIPE_TECH_ROLE: - schema_owner: # NOT SUPPORTED in tech_role.yaml - - SOURCE_STRIPE.STRIPE_WHY -``` - -SnowDDL's schema only allows: `grants`, `future_grants`, `account_grants`, `comment` - -**Role grants in technical roles don't work:** -```yaml -# tech_role.yaml - This ALSO won't work! -DLT_STRIPE_TECH_ROLE: - grants: - ROLE: # NOT a valid object type for grants - - SOURCE_STRIPE__STRIPE_WHY__OWNER__S_ROLE -``` - -Grant keys must be `OBJECT_TYPE:PRIVILEGE`, not just `ROLE`. - -### Complete DLT Permission Configuration - -**Technical Role** (`tech_role.yaml`): -```yaml -DLT_STRIPE_TECH_ROLE: - comment: DLT Stripe pipeline permissions - future_grants: - FILE_FORMAT:USAGE: - - SOURCE_STRIPE - SEQUENCE:USAGE: - - SOURCE_STRIPE - STAGE:USAGE,READ,WRITE: - - SOURCE_STRIPE - TABLE:SELECT,INSERT,UPDATE,DELETE,TRUNCATE,REFERENCES: - - SOURCE_STRIPE - VIEW:SELECT: - - SOURCE_STRIPE - grants: - DATABASE:USAGE,CREATE SCHEMA,MONITOR,MODIFY: - - SOURCE_STRIPE - SCHEMA:USAGE,MODIFY,MONITOR,CREATE TABLE,CREATE VIEW,CREATE FILE FORMAT,CREATE STAGE,CREATE SEQUENCE: - - SOURCE_STRIPE.STRIPE_WHY - WAREHOUSE:USAGE,MONITOR,OPERATE: - - DLT -``` - -**Business Role** (`business_role.yaml`): -```yaml -DLT_STRIPE_ROLE: - comment: DLT Stripe pipeline with schema owner access for table stages - tech_roles: - - DLT_STRIPE_TECH_ROLE - schema_owner: - - SOURCE_STRIPE.STRIPE_WHY # CRITICAL for table stage access - warehouse_usage: - - DLT -``` - -### Verifying the Configuration - -**Check role grants:** -```bash -snow sql --query "SHOW GRANTS OF ROLE SOURCE_STRIPE__STRIPE_WHY__OWNER__S_ROLE" -# Should show: granted to DLT_STRIPE_ROLE__B_ROLE -``` - -**Check table ownership:** -```bash -snow sql --query "SELECT TABLE_NAME, TABLE_OWNER FROM SOURCE_STRIPE.INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA = 'STRIPE_WHY'" -# Should show: TABLE_OWNER = SOURCE_STRIPE__STRIPE_WHY__OWNER__S_ROLE -``` - -**Check user inheritance:** -```bash -snow sql --query "SHOW GRANTS TO USER DLT" -# Should include: DLT_STRIPE_TECH_ROLE__T_ROLE -# User role should inherit from business role which has schema_owner -``` - -## Permission Model Patterns - -### One-Time Ownership Migration for Existing Schemas - -When adding `schema_owner` to a business role for a schema that already has objects, -existing objects need a one-time ownership transfer. SnowDDL only grants future ownership, -not retroactive ownership. - -**Problem**: DBT or other tools create objects BEFORE schema_owner is configured. -These objects remain owned by the original creator (often ACCOUNTADMIN). - -**Symptom**: After adding `schema_owner` and running SnowDDL apply: -``` -SQL access control error: Insufficient privileges to operate on table 'MY_TABLE' -SQL access control error: View 'MY_VIEW' already exists, but current role has no privileges -``` - -**Solution**: One-time ownership transfer after SnowDDL creates the schema owner role: - -```sql --- Transfer table ownership -GRANT OWNERSHIP ON ALL TABLES IN SCHEMA DB_NAME.SCHEMA_NAME - TO ROLE DB_NAME__SCHEMA_NAME__OWNER__S_ROLE COPY CURRENT GRANTS; - --- Transfer view ownership -GRANT OWNERSHIP ON ALL VIEWS IN SCHEMA DB_NAME.SCHEMA_NAME - TO ROLE DB_NAME__SCHEMA_NAME__OWNER__S_ROLE COPY CURRENT GRANTS; - --- If needed: procedures, functions, etc. -GRANT OWNERSHIP ON ALL PROCEDURES IN SCHEMA DB_NAME.SCHEMA_NAME - TO ROLE DB_NAME__SCHEMA_NAME__OWNER__S_ROLE COPY CURRENT GRANTS; -``` - -**Real example** (PROJ_STRIPE.PROJ_STRIPE for DBT): -```sql -GRANT OWNERSHIP ON ALL TABLES IN SCHEMA PROJ_STRIPE.PROJ_STRIPE - TO ROLE PROJ_STRIPE__PROJ_STRIPE__OWNER__S_ROLE COPY CURRENT GRANTS; - -GRANT OWNERSHIP ON ALL VIEWS IN SCHEMA PROJ_STRIPE.PROJ_STRIPE - TO ROLE PROJ_STRIPE__PROJ_STRIPE__OWNER__S_ROLE COPY CURRENT GRANTS; -``` - -**After migration**: Future objects are automatically owned by the schema owner role -through SnowDDL's future ownership grants. - -### Default SCHEMA_OWNER Model - -The default SnowDDL permission model creates schema owner roles with future OWNERSHIP grants: - -``` -When a table is created in SOURCE_STRIPE.STRIPE_WHY: -1. Creator initially owns it -2. Future ownership grant transfers ownership to SOURCE_STRIPE__STRIPE_WHY__OWNER__S_ROLE -3. Original creator loses table stage access -``` - -**Impact on data loaders**: DLT creates tables, but loses ownership, so it can't load data. - -**Solution**: Grant the schema owner role to DLT's business role (see above). - -### Alternative: Custom Permission Model (Not Recommended) - -You CAN create a custom permission model that doesn't transfer ownership: - -```yaml -# snowddl/permission_model.yaml -dlt_loader: - ruleset: SCHEMA_OWNER - owner_future_grants: - TABLE: [SELECT, INSERT, UPDATE, DELETE, TRUNCATE, REFERENCES] # No OWNERSHIP! -``` - -**Why this isn't recommended**: -1. Mixed ownership creates complexity -2. SnowDDL prefers consistent ownership patterns -3. The schema_owner solution on business roles is cleaner - -## Separating Read vs Write Access - -### Pattern: Source Database Read-Only for Transformers - -```yaml -# tech_role.yaml -DBT_STRIPE_ROLE: - comment: dbt role - READ-ONLY access to source, WRITE to project - future_grants: - TABLE:SELECT: - - SOURCE_STRIPE # Read-only - TABLE:SELECT,INSERT,UPDATE,DELETE,TRUNCATE: - - PROJ_STRIPE # Full write - grants: - SCHEMA:USAGE: - - SOURCE_STRIPE.STRIPE_WHY # Just USAGE, no CREATE - SCHEMA:USAGE,MODIFY,CREATE TABLE,CREATE VIEW: - - PROJ_STRIPE.PROJ_STRIPE # Full write -``` - -**Key principle**: Loaders write to SOURCE_*, transformers write to PROJ_*. - -## Git Patterns - -### Commit Messages -``` -type: short description - -Longer description if needed. - -Types: feat, fix, docs, refactor, test, chore -``` - -### Branch Names -``` -feature/description -fix/issue-description -docs/what-changed -``` diff --git a/docs/llm-context/README.md b/docs/llm-context/README.md deleted file mode 100644 index 056228a..0000000 --- a/docs/llm-context/README.md +++ /dev/null @@ -1,56 +0,0 @@ -# LLM Context Configuration - -This directory contains context files for LLMs (Claude, GPT, etc.) working with the SnowTower codebase. - -## Files - -| File | Purpose | -|------|---------| -| [CLAUDE.md](CLAUDE.md) | Detailed instructions for Claude Code | -| [CONTEXT.md](CONTEXT.md) | Project context and domain knowledge | -| [PATTERNS.md](PATTERNS.md) | Code patterns and conventions to follow | - -## Quick Start - -### For Claude Code Users - -The root `CLAUDE.md` file is automatically loaded by Claude Code. It contains: -- Project overview and architecture -- Key commands and workflows -- Code patterns to follow -- Safety guidelines - -### For Other LLMs - -Copy the contents of `CLAUDE.md` into your system prompt or context window. - -## Project-Specific Instructions - -When working with this codebase, LLMs should: - -1. **Always use UV for Python** - Never use pip directly -2. **Follow the command pattern** - Scripts go in `scripts/`, wrappers in `src/management_cli.py` -3. **Use load_dotenv()** - Every script must load environment variables -4. **Check before modifying** - Run `uv run snowddl-plan` before any changes - -## Key Directories - -``` -snowtower-snowddl/ -├── snowddl/ # YAML infrastructure definitions (MODIFY WITH CARE) -├── src/ # Python source code -│ ├── snowddl_core/ # OOP framework for SnowDDL -│ └── management_cli.py # CLI entry points -├── scripts/ # Management scripts -├── docs/ # Documentation -└── tests/ # Test files -``` - -## Safety Guidelines - -LLMs working with this codebase should: - -- **Never run `snowddl-apply` without user confirmation** -- **Always show plan output before applying changes** -- **Never modify production credentials** -- **Always use `--dry-run` when available** diff --git a/pyproject.toml b/pyproject.toml index 9433c7d..69490c7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "snowtower" -version = "0.2.0" +version = "0.3.0" description = "SnowTower - Snowflake Infrastructure Management Platform using Infrastructure as Code" readme = "README.md" requires-python = ">=3.10" @@ -50,6 +50,7 @@ dev = [ # API documentation generation "mkdocstrings[python]>=0.24.0", "mkdocstrings-python>=1.8.0", + "detect-secrets>=1.5.0", ] [build-system] @@ -85,6 +86,7 @@ manage-security = "src.management_cli:security" manage-backup = "src.management_cli:backup" apply-schema-grants = "src.management_cli:apply_schema_grants" validate-schema-grants = "src.management_cli:validate_schema_grants" +validate-config = "src.management_cli:validate_config" test-schema-mgmt = "scripts.test_schema_management:main" # === MONITORING & OBSERVABILITY === @@ -100,6 +102,10 @@ util-diagnose-auth = "src.diagnose_auth:main" util-fix-auth = "src.fix_auth:main" generate-rsa-batch = "scripts.generate_rsa_keys_batch:main" +# === TERRAFORM GENERATION === +# Convert SnowDDL YAML to Terraform HCL +generate-terraform = "src.management_cli:generate_terraform" + # === AUTOMATION === # GitHub issue to SnowDDL automation with PR creation github-to-snowddl = "src.management_cli:github_to_snowddl" diff --git a/scripts/generate_terraform.py b/scripts/generate_terraform.py new file mode 100644 index 0000000..ff9528c --- /dev/null +++ b/scripts/generate_terraform.py @@ -0,0 +1,753 @@ +from dotenv import load_dotenv + +load_dotenv() + +""" +Generate Terraform HCL files from SnowDDL YAML configurations. + +Converts SnowDDL YAML infrastructure definitions into Terraform-compatible +HCL or JSON files for the Snowflake-Labs/snowflake provider. + +Usage: + uv run generate-terraform # Print to stdout + uv run generate-terraform --output ./terraform/ # Write separate .tf files + uv run generate-terraform --format json # JSON output +""" + +import argparse +import json +import re +import sys +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Dict, List, Optional, Tuple + +import yaml + + +# --------------------------------------------------------------------------- +# Custom YAML loader that handles the !decrypt tag gracefully +# --------------------------------------------------------------------------- + +class SafeLoaderWithDecrypt(yaml.SafeLoader): + """YAML loader that treats !decrypt tagged values as placeholder strings.""" + pass + + +def _decrypt_constructor(loader: yaml.SafeLoader, node: yaml.ScalarNode) -> str: + """Return a placeholder for encrypted values.""" + return "" + + +SafeLoaderWithDecrypt.add_constructor("!decrypt", _decrypt_constructor) + + +# --------------------------------------------------------------------------- +# Constants +# --------------------------------------------------------------------------- + +WAREHOUSE_SIZE_MAP = { + "X-Small": "XSMALL", + "x-small": "XSMALL", + "xsmall": "XSMALL", + "Small": "SMALL", + "small": "SMALL", + "Medium": "MEDIUM", + "medium": "MEDIUM", + "Large": "LARGE", + "large": "LARGE", + "X-Large": "XLARGE", + "x-large": "XLARGE", + "xlarge": "XLARGE", + "2X-Large": "XXLARGE", + "2x-large": "XXLARGE", + "3X-Large": "XXXLARGE", + "3x-large": "XXXLARGE", + "4X-Large": "X4LARGE", + "4x-large": "X4LARGE", +} + +HEADER = """\ +# Generated by SnowTower - DO NOT EDIT MANUALLY +# Source: snowddl/ YAML configurations +""" + +PROVIDER_BLOCK = """\ +terraform { + required_providers { + snowflake = { + source = "Snowflake-Labs/snowflake" + version = "~> 1.0" + } + } +} +""" + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +def to_tf_name(name: str) -> str: + """Convert a Snowflake object name to a valid Terraform resource name. + + Rules: + - Lowercase + - Replace non-alphanumeric characters with underscores + - Strip leading/trailing underscores + - Collapse consecutive underscores + """ + result = name.lower() + result = re.sub(r"[^a-z0-9]", "_", result) + result = re.sub(r"_+", "_", result) + result = result.strip("_") + return result + + +def hcl_value(value: Any) -> str: + """Format a Python value as an HCL literal.""" + if isinstance(value, bool): + return "true" if value else "false" + if isinstance(value, int): + return str(value) + if isinstance(value, float): + return str(value) + if isinstance(value, str): + # Escape backslashes and quotes inside the string + escaped = value.replace("\\", "\\\\").replace('"', '\\"') + return f'"{escaped}"' + if isinstance(value, list): + items = ", ".join(hcl_value(v) for v in value) + return f"[{items}]" + return f'"{value}"' + + +def hcl_block(resource_type: str, resource_name: str, attrs: Dict[str, Any], + import_id: Optional[str] = None) -> str: + """Render a single Terraform resource block as HCL. + + Parameters + ---------- + resource_type : str + e.g. ``snowflake_user`` + resource_name : str + The Terraform resource name (snake_case) + attrs : dict + Key/value pairs for the resource body + import_id : str, optional + If provided an ``import {}`` block is emitted before the resource. + """ + lines: List[str] = [] + + if import_id is not None: + lines.append(f'import {{') + lines.append(f' to = {resource_type}.{resource_name}') + lines.append(f' id = "{import_id}"') + lines.append(f'}}') + lines.append("") + + lines.append(f'resource "{resource_type}" "{resource_name}" {{') + max_key_len = max((len(k) for k in attrs), default=0) + + for key, value in attrs.items(): + if isinstance(value, dict): + # Nested block (e.g. triggers in resource monitors) + lines.append(f" {key} {{") + for nk, nv in value.items(): + lines.append(f" {nk:{max_key_len}} = {hcl_value(nv)}") + lines.append(" }") + else: + lines.append(f" {key:{max_key_len}} = {hcl_value(value)}") + + lines.append("}") + return "\n".join(lines) + + +def load_yaml(path: Path) -> Optional[Dict]: + """Load a YAML file, returning None if missing or empty.""" + if not path.exists(): + return None + with open(path) as f: + data = yaml.load(f, Loader=SafeLoaderWithDecrypt) + return data if isinstance(data, dict) else None + + +# --------------------------------------------------------------------------- +# Generators -- each returns a list of HCL strings +# --------------------------------------------------------------------------- + +def generate_users(snowddl_dir: Path) -> List[str]: + """Generate snowflake_user resources from user.yaml.""" + data = load_yaml(snowddl_dir / "user.yaml") + if not data: + return [] + + blocks: List[str] = [] + for name, cfg in sorted(data.items()): + attrs: Dict[str, Any] = {"name": name} + for yaml_key, tf_key in [ + ("email", "email"), + ("first_name", "first_name"), + ("last_name", "last_name"), + ("default_role", "default_role"), + ("comment", "comment"), + ("type", "user_type"), + ]: + if yaml_key in cfg: + attrs[tf_key] = cfg[yaml_key] + + if "rsa_public_key" in cfg: + raw_key = cfg["rsa_public_key"].strip() + attrs["rsa_public_key"] = raw_key + + tf_name = to_tf_name(name) + blocks.append(hcl_block("snowflake_user", tf_name, attrs, import_id=name)) + + return blocks + + +def generate_warehouses(snowddl_dir: Path) -> List[str]: + """Generate snowflake_warehouse resources from warehouse.yaml.""" + data = load_yaml(snowddl_dir / "warehouse.yaml") + if not data: + return [] + + blocks: List[str] = [] + for name, cfg in sorted(data.items()): + size_raw = cfg.get("size", "X-Small") + size_tf = WAREHOUSE_SIZE_MAP.get(size_raw, size_raw.upper().replace("-", "")) + + attrs: Dict[str, Any] = { + "name": name, + "warehouse_size": size_tf, + } + + if "auto_suspend" in cfg: + attrs["auto_suspend"] = cfg["auto_suspend"] + attrs["auto_resume"] = cfg.get("auto_resume", True) + + for yaml_key, tf_key in [ + ("comment", "comment"), + ("resource_monitor", "resource_monitor"), + ("min_cluster_count", "min_cluster_count"), + ("max_cluster_count", "max_cluster_count"), + ]: + if yaml_key in cfg: + attrs[tf_key] = cfg[yaml_key] + + tf_name = to_tf_name(name) + blocks.append(hcl_block("snowflake_warehouse", tf_name, attrs, import_id=name)) + + return blocks + + +def generate_business_roles(snowddl_dir: Path) -> Tuple[List[str], List[str]]: + """Generate snowflake_account_role and grant resources from business_role.yaml. + + Returns (role_blocks, grant_blocks). + """ + data = load_yaml(snowddl_dir / "business_role.yaml") + if not data: + return [], [] + + role_blocks: List[str] = [] + grant_blocks: List[str] = [] + + for name, cfg in sorted(data.items()): + sf_role_name = f"{name}__B_ROLE" + tf_name = to_tf_name(sf_role_name) + + attrs = { + "name": sf_role_name, + } + if "comment" in cfg: + attrs["comment"] = cfg["comment"] + + role_blocks.append(hcl_block("snowflake_account_role", tf_name, attrs, + import_id=sf_role_name)) + + # Tech role inheritance grants + for tech_role in cfg.get("tech_roles", []): + tech_sf_name = f"{tech_role}__T_ROLE" + grant_tf_name = to_tf_name(f"{sf_role_name}_inherits_{tech_sf_name}") + grant_attrs = { + "role_name": tech_sf_name, + "parent_role_name": sf_role_name, + } + grant_blocks.append(hcl_block( + "snowflake_grant_account_role", grant_tf_name, grant_attrs)) + + # Warehouse usage grants + for wh in cfg.get("warehouse_usage", []): + grant_tf_name = to_tf_name(f"{sf_role_name}_wh_{wh}") + grant_attrs = { + "account_role_name": sf_role_name, + "privileges": ["USAGE"], + "on_account_object": {"object_type": "WAREHOUSE", "object_name": wh}, + } + grant_blocks.append(hcl_block( + "snowflake_grant_privileges_to_account_role", grant_tf_name, grant_attrs)) + + # Schema owner grants + for schema_ref in cfg.get("schema_owner", []): + parts = schema_ref.split(".") + if len(parts) == 2: + db, schema = parts + owner_role = f"{db}__{schema}__OWNER__S_ROLE" + grant_tf_name = to_tf_name(f"{sf_role_name}_owner_{db}_{schema}") + grant_attrs = { + "role_name": owner_role, + "parent_role_name": sf_role_name, + } + grant_blocks.append(hcl_block( + "snowflake_grant_account_role", grant_tf_name, grant_attrs)) + + return role_blocks, grant_blocks + + +def _parse_grant_key(grant_key: str) -> Tuple[str, List[str]]: + """Parse a SnowDDL grant key like ``DATABASE:USAGE,CREATE SCHEMA``. + + Returns (object_type, privilege_list). + """ + parts = grant_key.split(":", 1) + obj_type = parts[0] + privileges = [p.strip() for p in parts[1].split(",")] + return obj_type, privileges + + +def generate_tech_roles(snowddl_dir: Path) -> Tuple[List[str], List[str]]: + """Generate snowflake_account_role and grant resources from tech_role.yaml. + + Returns (role_blocks, grant_blocks). + """ + data = load_yaml(snowddl_dir / "tech_role.yaml") + if not data: + return [], [] + + role_blocks: List[str] = [] + grant_blocks: List[str] = [] + + for name, cfg in sorted(data.items()): + sf_role_name = f"{name}__T_ROLE" + tf_name = to_tf_name(sf_role_name) + + attrs = {"name": sf_role_name} + if "comment" in cfg: + attrs["comment"] = cfg["comment"] + + role_blocks.append(hcl_block("snowflake_account_role", tf_name, attrs, + import_id=sf_role_name)) + + # Regular grants + for grant_key, targets in (cfg.get("grants") or {}).items(): + obj_type, privileges = _parse_grant_key(grant_key) + if not targets: + continue + for target in targets: + grant_tf_name = to_tf_name( + f"{sf_role_name}_grant_{obj_type}_{target}_{'_'.join(privileges)}" + ) + grant_attrs: Dict[str, Any] = { + "account_role_name": sf_role_name, + "privileges": privileges, + } + + if obj_type == "SCHEMA": + # target is DATABASE.SCHEMA + grant_attrs["on_schema"] = { + "schema_name": f'"{target}"', + } + elif obj_type in ("DATABASE", "WAREHOUSE"): + grant_attrs["on_account_object"] = { + "object_type": obj_type, + "object_name": target, + } + else: + grant_attrs["on_account_object"] = { + "object_type": obj_type, + "object_name": target, + } + + grant_blocks.append(hcl_block( + "snowflake_grant_privileges_to_account_role", + grant_tf_name, grant_attrs)) + + # Future grants + for grant_key, targets in (cfg.get("future_grants") or {}).items(): + obj_type, privileges = _parse_grant_key(grant_key) + if not targets: + continue + for target in targets: + grant_tf_name = to_tf_name( + f"{sf_role_name}_future_{obj_type}_{target}_{'_'.join(privileges)}" + ) + grant_attrs = { + "account_role_name": sf_role_name, + "privileges": privileges, + "on_schema_object": { + "object_type_plural": f"{obj_type}S", + "in_database": target, + }, + "all_privileges": False, + "with_grant_option": False, + } + + grant_blocks.append(hcl_block( + "snowflake_grant_privileges_to_account_role", + grant_tf_name, grant_attrs)) + + return role_blocks, grant_blocks + + +def generate_network_policies(snowddl_dir: Path) -> List[str]: + """Generate snowflake_network_policy resources from network_policy.yaml.""" + data = load_yaml(snowddl_dir / "network_policy.yaml") + if not data: + return [] + + blocks: List[str] = [] + for name, cfg in sorted(data.items()): + sf_name = name.upper() + attrs: Dict[str, Any] = {"name": sf_name} + + if "allowed_ip_list" in cfg: + attrs["allowed_ip_list"] = cfg["allowed_ip_list"] + if "comment" in cfg: + attrs["comment"] = cfg["comment"] + + tf_name = to_tf_name(name) + blocks.append(hcl_block("snowflake_network_policy", tf_name, attrs, + import_id=sf_name)) + + return blocks + + +def generate_authentication_policies(snowddl_dir: Path) -> List[str]: + """Generate snowflake_authentication_policy resources.""" + data = load_yaml(snowddl_dir / "authentication_policy.yaml") + if not data: + return [] + + blocks: List[str] = [] + for name, cfg in sorted(data.items()): + sf_name = name.upper() + attrs: Dict[str, Any] = {"name": sf_name} + + for yaml_key, tf_key in [ + ("comment", "comment"), + ("client_types", "client_types"), + ("mfa_authentication_methods", "mfa_authentication_methods"), + ("mfa_enrollment", "mfa_enrollment"), + ("authentication_methods", "authentication_methods"), + ("security_integrations", "security_integrations"), + ]: + if yaml_key in cfg: + attrs[tf_key] = cfg[yaml_key] + + tf_name = to_tf_name(name) + blocks.append(hcl_block("snowflake_authentication_policy", tf_name, attrs, + import_id=sf_name)) + + return blocks + + +def generate_password_policies(snowddl_dir: Path) -> List[str]: + """Generate snowflake_password_policy resources.""" + data = load_yaml(snowddl_dir / "password_policy.yaml") + if not data: + return [] + + blocks: List[str] = [] + for name, cfg in sorted(data.items()): + sf_name = name.upper() + attrs: Dict[str, Any] = {"name": sf_name} + + # Map all password_* fields and other known fields + for key, value in sorted(cfg.items()): + if key == "comment_in_sql": + continue # SnowDDL-specific, not relevant for Terraform + attrs[key] = value + + tf_name = to_tf_name(name) + blocks.append(hcl_block("snowflake_password_policy", tf_name, attrs, + import_id=sf_name)) + + return blocks + + +def generate_session_policies(snowddl_dir: Path) -> List[str]: + """Generate snowflake_session_policy resources.""" + data = load_yaml(snowddl_dir / "session_policy.yaml") + if not data: + return [] + + blocks: List[str] = [] + for name, cfg in sorted(data.items()): + sf_name = name.upper() + attrs: Dict[str, Any] = {"name": sf_name} + + for key, value in sorted(cfg.items()): + if key == "comment_in_sql": + continue + attrs[key] = value + + tf_name = to_tf_name(name) + blocks.append(hcl_block("snowflake_session_policy", tf_name, attrs, + import_id=sf_name)) + + return blocks + + +def generate_resource_monitors(snowddl_dir: Path) -> List[str]: + """Generate snowflake_resource_monitor resources from resource_monitor.yaml.""" + data = load_yaml(snowddl_dir / "resource_monitor.yaml") + if not data: + return [] + + blocks: List[str] = [] + for name, cfg in sorted(data.items()): + attrs: Dict[str, Any] = {"name": name} + + if "credit_quota" in cfg: + attrs["credit_quota"] = cfg["credit_quota"] + if "frequency" in cfg: + attrs["frequency"] = cfg["frequency"] + + # Parse triggers into separate lists + triggers = cfg.get("triggers", {}) + notify_triggers: List[int] = [] + suspend_triggers: List[int] = [] + suspend_immediate_triggers: List[int] = [] + + for threshold, action in sorted(triggers.items()): + action_upper = str(action).upper() + if action_upper == "NOTIFY": + notify_triggers.append(int(threshold)) + elif action_upper == "SUSPEND": + suspend_triggers.append(int(threshold)) + elif action_upper == "SUSPEND_IMMEDIATE": + suspend_immediate_triggers.append(int(threshold)) + + if notify_triggers: + attrs["notify_triggers"] = notify_triggers + if suspend_triggers: + attrs["suspend_triggers"] = suspend_triggers + if suspend_immediate_triggers: + attrs["suspend_immediate_triggers"] = suspend_immediate_triggers + + tf_name = to_tf_name(name) + blocks.append(hcl_block("snowflake_resource_monitor", tf_name, attrs, + import_id=name)) + + return blocks + + +def generate_databases(snowddl_dir: Path) -> List[str]: + """Generate snowflake_database resources from database directories.""" + blocks: List[str] = [] + + for params_path in sorted(snowddl_dir.glob("*/params.yaml")): + db_name = params_path.parent.name + cfg = load_yaml(params_path) or {} + + attrs: Dict[str, Any] = {"name": db_name} + + if "comment" in cfg: + attrs["comment"] = cfg["comment"] + if cfg.get("is_sandbox"): + attrs["comment"] = attrs.get("comment", f"Sandbox database {db_name}") + + tf_name = to_tf_name(db_name) + blocks.append(hcl_block("snowflake_database", tf_name, attrs, + import_id=db_name)) + + return blocks + + +# --------------------------------------------------------------------------- +# Output formatting +# --------------------------------------------------------------------------- + +@dataclass +class TerraformOutput: + """Holds all generated Terraform sections.""" + main: str = "" + users: List[str] = field(default_factory=list) + warehouses: List[str] = field(default_factory=list) + roles: List[str] = field(default_factory=list) + grants: List[str] = field(default_factory=list) + policies: List[str] = field(default_factory=list) + databases: List[str] = field(default_factory=list) + resource_monitors: List[str] = field(default_factory=list) + + +def generate_all(snowddl_dir: Path) -> TerraformOutput: + """Run all generators and return structured output.""" + output = TerraformOutput() + output.main = PROVIDER_BLOCK + + # Users + output.users = generate_users(snowddl_dir) + + # Warehouses + output.warehouses = generate_warehouses(snowddl_dir) + + # Business roles + b_role_blocks, b_grant_blocks = generate_business_roles(snowddl_dir) + + # Tech roles + t_role_blocks, t_grant_blocks = generate_tech_roles(snowddl_dir) + + output.roles = b_role_blocks + t_role_blocks + output.grants = b_grant_blocks + t_grant_blocks + + # Policies (network, authentication, password, session) + output.policies = ( + generate_network_policies(snowddl_dir) + + generate_authentication_policies(snowddl_dir) + + generate_password_policies(snowddl_dir) + + generate_session_policies(snowddl_dir) + ) + + # Resource monitors + output.resource_monitors = generate_resource_monitors(snowddl_dir) + + # Databases + output.databases = generate_databases(snowddl_dir) + + return output + + +def render_section(header_comment: str, blocks: List[str]) -> str: + """Combine blocks into a single string with a section header.""" + if not blocks: + return "" + lines = [HEADER, f"# {header_comment}", ""] + lines.append("\n\n".join(blocks)) + lines.append("") + return "\n".join(lines) + + +def write_to_directory(output: TerraformOutput, out_dir: Path) -> None: + """Write separate .tf files into the output directory.""" + out_dir.mkdir(parents=True, exist_ok=True) + + files = { + "main.tf": HEADER + "\n" + output.main, + "users.tf": render_section("User resources", output.users), + "warehouses.tf": render_section("Warehouse resources", output.warehouses), + "roles.tf": render_section("Role resources (business + tech)", output.roles), + "grants.tf": render_section("Grant resources", output.grants), + "policies.tf": render_section( + "Policy resources (network, authentication, password, session)", + output.policies), + "databases.tf": render_section("Database resources", output.databases), + "resource_monitors.tf": render_section( + "Resource monitor resources", output.resource_monitors), + } + + for filename, content in files.items(): + if not content.strip(): + continue + filepath = out_dir / filename + filepath.write_text(content) + print(f" Wrote {filepath}") + + +def write_to_stdout(output: TerraformOutput, fmt: str) -> None: + """Print combined output to stdout.""" + if fmt == "json": + # Collect a simple representation for JSON output + combined = { + "main": output.main, + "users": output.users, + "warehouses": output.warehouses, + "roles": output.roles, + "grants": output.grants, + "policies": output.policies, + "databases": output.databases, + "resource_monitors": output.resource_monitors, + } + print(json.dumps(combined, indent=2)) + return + + # HCL to stdout -- combine all sections + sections = [ + (HEADER + "\n" + output.main), + render_section("User resources", output.users), + render_section("Warehouse resources", output.warehouses), + render_section("Role resources (business + tech)", output.roles), + render_section("Grant resources", output.grants), + render_section("Policy resources (network, auth, password, session)", + output.policies), + render_section("Database resources", output.databases), + render_section("Resource monitor resources", output.resource_monitors), + ] + + print("\n".join(s for s in sections if s.strip())) + + +# --------------------------------------------------------------------------- +# CLI +# --------------------------------------------------------------------------- + +def main() -> None: + parser = argparse.ArgumentParser( + description="Generate Terraform HCL files from SnowDDL YAML configurations.", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog="""\ +Examples: + uv run generate-terraform # Print HCL to stdout + uv run generate-terraform --output ./terraform/ # Write separate .tf files + uv run generate-terraform --format json # JSON output to stdout +""", + ) + parser.add_argument( + "--output", "-o", + type=str, + default=None, + help="Output directory for .tf files. If omitted, prints to stdout.", + ) + parser.add_argument( + "--format", "-f", + choices=["hcl", "json"], + default="hcl", + dest="fmt", + help="Output format (default: hcl).", + ) + parser.add_argument( + "--snowddl-dir", + type=str, + default=None, + help="Path to snowddl/ directory (auto-detected from project root).", + ) + args = parser.parse_args() + + # Resolve snowddl directory + if args.snowddl_dir: + snowddl_dir = Path(args.snowddl_dir) + else: + # Walk up from this script to find snowddl/ + project_root = Path(__file__).resolve().parent.parent + snowddl_dir = project_root / "snowddl" + + if not snowddl_dir.is_dir(): + print(f"Error: snowddl directory not found at {snowddl_dir}", file=sys.stderr) + sys.exit(1) + + print(f"Reading SnowDDL configs from: {snowddl_dir}", file=sys.stderr) + + output = generate_all(snowddl_dir) + + if args.output: + out_dir = Path(args.output) + print(f"Writing Terraform files to: {out_dir}", file=sys.stderr) + write_to_directory(output, out_dir) + print(f"\nDone. Generated Terraform files in {out_dir}", file=sys.stderr) + else: + write_to_stdout(output, args.fmt) + + +if __name__ == "__main__": + main() diff --git a/scripts/validate_config.py b/scripts/validate_config.py new file mode 100644 index 0000000..c30aecb --- /dev/null +++ b/scripts/validate_config.py @@ -0,0 +1,813 @@ +#!/usr/bin/env python3 +""" +SnowDDL YAML Configuration Validator. + +Validates all SnowDDL YAML configuration files before deployment. +Performs syntax checking, field validation, and cross-reference integrity checks. + +Usage: + uv run validate-config # Validate all configs + uv run validate-config --strict # Fail on warnings too + uv run validate-config --quiet # Only show errors + uv run validate-config snowddl/user.yaml # Validate specific file(s) +""" + +from dotenv import load_dotenv + +load_dotenv() + +import argparse +import ipaddress +import re +import sys +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Dict, List, Optional, Set, Tuple + +import yaml + + +# --------------------------------------------------------------------------- +# Constants +# --------------------------------------------------------------------------- + +VALID_WAREHOUSE_SIZES: Set[str] = { + "X-Small", + "Small", + "Medium", + "Large", + "X-Large", + "2X-Large", + "3X-Large", + "4X-Large", + "5X-Large", + "6X-Large", +} + +# Also accept uppercased/alternative forms that Snowflake recognises +VALID_WAREHOUSE_SIZES_NORMALISED: Set[str] = { + s.upper().replace("-", "") for s in VALID_WAREHOUSE_SIZES +} + +VALID_GRANT_OBJECT_TYPES: Set[str] = { + "DATABASE", + "SCHEMA", + "TABLE", + "VIEW", + "WAREHOUSE", + "STAGE", + "FUNCTION", + "PROCEDURE", + "FILE_FORMAT", + "SEQUENCE", +} + +SYSTEM_ROLES: Set[str] = { + "ACCOUNTADMIN", + "SYSADMIN", + "USERADMIN", + "SECURITYADMIN", + "ORGADMIN", + "PUBLIC", +} + +VALID_USER_TYPES: Set[str] = {"PERSON", "SERVICE"} + +# SnowDDL appends __B_ROLE to business role names +BUSINESS_ROLE_SUFFIX = "__B_ROLE" + + +# --------------------------------------------------------------------------- +# Result tracking +# --------------------------------------------------------------------------- + + +@dataclass +class ValidationResult: + """Accumulates errors and warnings across all checks.""" + + errors: List[str] = field(default_factory=list) + warnings: List[str] = field(default_factory=list) + info: List[str] = field(default_factory=list) + + def error(self, message: str) -> None: + self.errors.append(message) + + def warning(self, message: str) -> None: + self.warnings.append(message) + + def ok(self, message: str) -> None: + self.info.append(message) + + @property + def has_errors(self) -> bool: + return len(self.errors) > 0 + + @property + def has_warnings(self) -> bool: + return len(self.warnings) > 0 + + +# --------------------------------------------------------------------------- +# YAML loading helpers +# --------------------------------------------------------------------------- + + +def _find_project_root() -> Path: + """Walk upwards from this script to find the project root (contains pyproject.toml).""" + current = Path(__file__).resolve().parent + for _ in range(10): + if (current / "pyproject.toml").exists(): + return current + current = current.parent + # Fallback: assume scripts/ is one level below root + return Path(__file__).resolve().parent.parent + + +def _load_yaml(path: Path) -> Tuple[Optional[Dict[str, Any]], Optional[str]]: + """Load and parse a YAML file. Returns (data, error_message).""" + if not path.exists(): + return None, f"File not found: {path}" + try: + with open(path, "r") as f: + data = yaml.safe_load(f) + if data is None: + data = {} + if not isinstance(data, dict): + return None, f"Expected a YAML mapping at top level, got {type(data).__name__}" + return data, None + except yaml.YAMLError as exc: + return None, f"YAML syntax error: {exc}" + + +# --------------------------------------------------------------------------- +# Per-file validators +# --------------------------------------------------------------------------- + + +def validate_user_yaml( + data: Dict[str, Any], + result: ValidationResult, + business_roles: Set[str], +) -> None: + """Validate user.yaml entries.""" + names_seen: Set[str] = set() + + for user_name, user_cfg in data.items(): + if not isinstance(user_cfg, dict): + result.error(f"User {user_name}: expected a mapping, got {type(user_cfg).__name__}") + continue + + # Duplicate check + upper_name = user_name.upper() + if upper_name in names_seen: + result.error(f"User {user_name}: duplicate user name") + names_seen.add(upper_name) + + # Type field + user_type = user_cfg.get("type") + if user_type is None: + result.error(f"User {user_name}: missing required 'type' field (PERSON or SERVICE)") + elif str(user_type).upper() not in VALID_USER_TYPES: + result.error( + f"User {user_name}: invalid type '{user_type}' (must be PERSON or SERVICE)" + ) + else: + user_type = str(user_type).upper() + + # PERSON-specific checks + if user_type == "PERSON": + if "email" not in user_cfg: + result.warning(f"User {user_name}: PERSON user should have 'email' field") + + # SERVICE-specific checks + if user_type == "SERVICE": + rsa_key = user_cfg.get("rsa_public_key", "") + if not rsa_key: + result.warning( + f"User {user_name}: SERVICE user should have 'rsa_public_key' field" + ) + elif "example" in str(rsa_key).lower() or "replace" in str(rsa_key).lower(): + result.warning(f"User {user_name}: RSA key appears to be a placeholder") + + # default_role cross-reference (deferred to cross-ref if business_roles provided) + default_role = user_cfg.get("default_role") + if default_role and business_roles: + role_upper = str(default_role).upper() + # Accept system roles, raw business role names, and names with __B_ROLE suffix + is_system = role_upper in SYSTEM_ROLES + is_business_raw = role_upper in business_roles + is_business_suffixed = role_upper.replace(BUSINESS_ROLE_SUFFIX, "") in business_roles + if not (is_system or is_business_raw or is_business_suffixed): + result.error( + f"User {user_name}: default_role '{default_role}' does not match any " + f"system role or business role in business_role.yaml" + ) + + count = len(data) + result.ok(f"{count} user{'s' if count != 1 else ''} validated") + + +def validate_business_role_yaml( + data: Dict[str, Any], + result: ValidationResult, + tech_roles: Set[str], + warehouses: Set[str], +) -> None: + """Validate business_role.yaml entries.""" + names_seen: Set[str] = set() + + for role_name, role_cfg in data.items(): + if not isinstance(role_cfg, dict): + result.error( + f"Business role {role_name}: expected a mapping, got {type(role_cfg).__name__}" + ) + continue + + upper_name = role_name.upper() + if upper_name in names_seen: + result.error(f"Business role {role_name}: duplicate role name") + names_seen.add(upper_name) + + # tech_roles references + for tr in role_cfg.get("tech_roles", []): + if tech_roles and str(tr).upper() not in tech_roles: + result.error( + f"Business role {role_name}: tech_role '{tr}' not found in tech_role.yaml" + ) + + # warehouse_usage references + for wh in role_cfg.get("warehouse_usage", []): + if warehouses and str(wh).upper() not in warehouses: + result.error( + f"Business role {role_name}: warehouse '{wh}' not found in warehouse.yaml" + ) + + # schema_owner format (DB.SCHEMA) + for so in role_cfg.get("schema_owner", []): + if not re.match(r"^[A-Za-z0-9_]+\.[A-Za-z0-9_]+$", str(so)): + result.error( + f"Business role {role_name}: schema_owner '{so}' is not valid " + f"DB.SCHEMA format" + ) + + count = len(data) + result.ok(f"{count} business role{'s' if count != 1 else ''} validated") + + +def validate_tech_role_yaml( + data: Dict[str, Any], + result: ValidationResult, + warehouses: Set[str], +) -> None: + """Validate tech_role.yaml entries.""" + names_seen: Set[str] = set() + + for role_name, role_cfg in data.items(): + if not isinstance(role_cfg, dict): + result.error( + f"Tech role {role_name}: expected a mapping, got {type(role_cfg).__name__}" + ) + continue + + upper_name = role_name.upper() + if upper_name in names_seen: + result.error(f"Tech role {role_name}: duplicate role name") + names_seen.add(upper_name) + + # Validate grant keys in both 'grants' and 'future_grants' + for section_name in ("grants", "future_grants"): + section = role_cfg.get(section_name, {}) + if not isinstance(section, dict): + continue + + for grant_key, targets in section.items(): + # Grant key format: OBJECT_TYPE:PRIVILEGE(S) + parts = str(grant_key).split(":", 1) + if len(parts) != 2 or not parts[1]: + result.error( + f"Tech role {role_name}: grant key '{grant_key}' in {section_name} " + f"must follow OBJECT_TYPE:PRIVILEGE format" + ) + continue + + obj_type = parts[0].upper() + if obj_type not in VALID_GRANT_OBJECT_TYPES: + result.error( + f"Tech role {role_name}: invalid object type '{parts[0]}' in " + f"grant key '{grant_key}' (valid: {', '.join(sorted(VALID_GRANT_OBJECT_TYPES))})" + ) + + # WAREHOUSE:USAGE cross-reference + if obj_type == "WAREHOUSE" and warehouses and isinstance(targets, list): + for wh in targets: + if str(wh).upper() not in warehouses: + result.error( + f"Tech role {role_name}: warehouse '{wh}' in " + f"{section_name} not found in warehouse.yaml" + ) + + count = len(data) + result.ok(f"{count} technical role{'s' if count != 1 else ''} validated") + + +def validate_warehouse_yaml( + data: Dict[str, Any], + result: ValidationResult, + resource_monitors: Set[str], +) -> None: + """Validate warehouse.yaml entries.""" + names_seen: Set[str] = set() + + for wh_name, wh_cfg in data.items(): + if not isinstance(wh_cfg, dict): + result.error( + f"Warehouse {wh_name}: expected a mapping, got {type(wh_cfg).__name__}" + ) + continue + + upper_name = wh_name.upper() + if upper_name in names_seen: + result.error(f"Warehouse {wh_name}: duplicate warehouse name") + names_seen.add(upper_name) + + # Size validation + size = wh_cfg.get("size") + if size is not None: + normalised = str(size).upper().replace("-", "") + if normalised not in VALID_WAREHOUSE_SIZES_NORMALISED: + result.error( + f"Warehouse {wh_name}: invalid size '{size}' " + f"(valid: {', '.join(sorted(VALID_WAREHOUSE_SIZES))})" + ) + + # auto_suspend validation + auto_suspend = wh_cfg.get("auto_suspend") + if auto_suspend is not None: + try: + val = int(auto_suspend) + if val < 0: + result.error( + f"Warehouse {wh_name}: auto_suspend must be a positive integer, got {val}" + ) + except (ValueError, TypeError): + result.error( + f"Warehouse {wh_name}: auto_suspend must be an integer, got '{auto_suspend}'" + ) + + # resource_monitor cross-reference + monitor = wh_cfg.get("resource_monitor") + if monitor and resource_monitors: + if str(monitor).upper() not in resource_monitors: + result.error( + f"Warehouse {wh_name}: resource_monitor '{monitor}' " + f"not found in resource_monitor.yaml" + ) + + count = len(data) + result.ok(f"{count} warehouse{'s' if count != 1 else ''} validated") + + +def validate_network_policy_yaml( + data: Dict[str, Any], + result: ValidationResult, +) -> None: + """Validate network_policy.yaml entries.""" + names_seen: Set[str] = set() + + for policy_name, policy_cfg in data.items(): + if not isinstance(policy_cfg, dict): + result.error( + f"Network policy {policy_name}: expected a mapping, got {type(policy_cfg).__name__}" + ) + continue + + upper_name = policy_name.upper() + if upper_name in names_seen: + result.error(f"Network policy {policy_name}: duplicate policy name") + names_seen.add(upper_name) + + # Validate CIDR entries in allowed_ip_list + for ip_entry in policy_cfg.get("allowed_ip_list", []): + try: + ipaddress.ip_network(str(ip_entry), strict=False) + except ValueError: + result.error( + f"Network policy {policy_name}: '{ip_entry}' is not valid CIDR notation" + ) + + # Also check blocked_ip_list if present + for ip_entry in policy_cfg.get("blocked_ip_list", []): + try: + ipaddress.ip_network(str(ip_entry), strict=False) + except ValueError: + result.error( + f"Network policy {policy_name}: blocked IP '{ip_entry}' is not valid CIDR notation" + ) + + count = len(data) + result.ok(f"{count} network polic{'ies' if count != 1 else 'y'} validated") + + +def validate_resource_monitor_yaml( + data: Dict[str, Any], + result: ValidationResult, +) -> None: + """Validate resource_monitor.yaml entries (basic structure check).""" + for monitor_name, monitor_cfg in data.items(): + if not isinstance(monitor_cfg, dict): + result.error( + f"Resource monitor {monitor_name}: expected a mapping, got {type(monitor_cfg).__name__}" + ) + continue + + credit_quota = monitor_cfg.get("credit_quota") + if credit_quota is not None: + try: + val = int(credit_quota) + if val <= 0: + result.error( + f"Resource monitor {monitor_name}: credit_quota must be positive, got {val}" + ) + except (ValueError, TypeError): + result.error( + f"Resource monitor {monitor_name}: credit_quota must be an integer" + ) + + count = len(data) + result.ok(f"{count} resource monitor{'s' if count != 1 else ''} validated") + + +# --------------------------------------------------------------------------- +# Output helpers +# --------------------------------------------------------------------------- + +# ANSI colour helpers for terminal output +_GREEN = "\033[32m" +_YELLOW = "\033[33m" +_RED = "\033[31m" +_RESET = "\033[0m" +_BOLD = "\033[1m" + + +def _print_file_results( + filename: str, + file_result: ValidationResult, + quiet: bool, +) -> None: + """Print validation results for a single file.""" + has_output = False + + if not quiet: + print(f"\n{_BOLD}{filename}{_RESET}") + has_output = True + + # Print info (success messages) + if not quiet: + for msg in file_result.info: + print(f" {_GREEN}\u2713{_RESET} {msg}") + + # Print warnings + if not quiet: + for msg in file_result.warnings: + print(f" {_YELLOW}\u26a0 WARNING:{_RESET} {msg}") + + # Print errors (always shown) + for msg in file_result.errors: + if not has_output: + print(f"\n{_BOLD}{filename}{_RESET}") + has_output = True + print(f" {_RED}\u2717 ERROR:{_RESET} {msg}") + + +# --------------------------------------------------------------------------- +# Main validation orchestrator +# --------------------------------------------------------------------------- + + +def run_validation(config_dir: Path, target_files: Optional[List[str]], strict: bool, quiet: bool) -> int: + """ + Run all validation checks and return exit code. + + Returns: + 0 on success (or warnings-only without --strict) + 1 on errors (or warnings with --strict) + """ + if not config_dir.is_dir(): + print(f"{_RED}Error:{_RESET} Config directory not found: {config_dir}") + return 1 + + if not quiet: + print(f"Validating {config_dir}/ configuration...") + + # ----------------------------------------------------------------------- + # 1. Load all YAML files (needed for cross-references even if targeting specific files) + # ----------------------------------------------------------------------- + + yaml_files = { + "user.yaml": config_dir / "user.yaml", + "business_role.yaml": config_dir / "business_role.yaml", + "tech_role.yaml": config_dir / "tech_role.yaml", + "warehouse.yaml": config_dir / "warehouse.yaml", + "network_policy.yaml": config_dir / "network_policy.yaml", + "resource_monitor.yaml": config_dir / "resource_monitor.yaml", + } + + loaded: Dict[str, Dict[str, Any]] = {} + syntax_result = ValidationResult() + + for name, path in yaml_files.items(): + if not path.exists(): + # Not every project will have all files; skip missing ones silently + loaded[name] = {} + continue + data, err = _load_yaml(path) + if err: + syntax_result.error(f"{name}: {err}") + loaded[name] = {} + else: + loaded[name] = data # type: ignore[assignment] + + # Print syntax errors first + if syntax_result.has_errors: + _print_file_results("YAML Syntax", syntax_result, quiet) + + # ----------------------------------------------------------------------- + # 2. Build cross-reference sets + # ----------------------------------------------------------------------- + + business_roles: Set[str] = {k.upper() for k in loaded.get("business_role.yaml", {})} + tech_roles: Set[str] = {k.upper() for k in loaded.get("tech_role.yaml", {})} + warehouses: Set[str] = {k.upper() for k in loaded.get("warehouse.yaml", {})} + resource_monitors: Set[str] = {k.upper() for k in loaded.get("resource_monitor.yaml", {})} + + # ----------------------------------------------------------------------- + # 3. Determine which files to validate + # ----------------------------------------------------------------------- + + if target_files: + # Resolve target files to their canonical names + files_to_validate: Set[str] = set() + for tf in target_files: + tf_path = Path(tf) + basename = tf_path.name + if basename in yaml_files: + files_to_validate.add(basename) + else: + # Try to match by stem + for known_name in yaml_files: + if known_name.startswith(tf_path.stem): + files_to_validate.add(known_name) + break + else: + print(f"{_YELLOW}\u26a0 WARNING:{_RESET} Unknown config file: {tf} (skipping)") + else: + files_to_validate = set(yaml_files.keys()) + + # ----------------------------------------------------------------------- + # 4. Run per-file validators + # ----------------------------------------------------------------------- + + all_results: List[Tuple[str, ValidationResult]] = [] + + validators = { + "user.yaml": lambda r: validate_user_yaml(loaded["user.yaml"], r, business_roles), + "business_role.yaml": lambda r: validate_business_role_yaml( + loaded["business_role.yaml"], r, tech_roles, warehouses + ), + "tech_role.yaml": lambda r: validate_tech_role_yaml( + loaded["tech_role.yaml"], r, warehouses + ), + "warehouse.yaml": lambda r: validate_warehouse_yaml( + loaded["warehouse.yaml"], r, resource_monitors + ), + "network_policy.yaml": lambda r: validate_network_policy_yaml( + loaded["network_policy.yaml"], r + ), + "resource_monitor.yaml": lambda r: validate_resource_monitor_yaml( + loaded["resource_monitor.yaml"], r + ), + } + + for filename in sorted(files_to_validate): + if filename not in validators: + continue + if not loaded.get(filename): + continue + + file_result = ValidationResult() + validators[filename](file_result) + all_results.append((filename, file_result)) + _print_file_results(filename, file_result, quiet) + + # ----------------------------------------------------------------------- + # 5. Cross-reference validation (only when validating all files) + # ----------------------------------------------------------------------- + + if not target_files: + xref_result = ValidationResult() + _cross_reference_checks(loaded, xref_result, business_roles, tech_roles, warehouses, resource_monitors) + if xref_result.errors or xref_result.warnings or xref_result.info: + if not quiet or xref_result.errors: + print(f"\n{_BOLD}Cross-reference checks:{_RESET}") + for msg in xref_result.info: + if not quiet: + print(f" {_GREEN}\u2713{_RESET} {msg}") + for msg in xref_result.warnings: + if not quiet: + print(f" {_YELLOW}\u26a0 WARNING:{_RESET} {msg}") + for msg in xref_result.errors: + print(f" {_RED}\u2717 ERROR:{_RESET} {msg}") + all_results.append(("cross-references", xref_result)) + + # ----------------------------------------------------------------------- + # 6. Summary + # ----------------------------------------------------------------------- + + total_errors = syntax_result.errors[:] + total_warnings = syntax_result.warnings[:] + for _, fr in all_results: + total_errors.extend(fr.errors) + total_warnings.extend(fr.warnings) + + error_count = len(total_errors) + warning_count = len(total_warnings) + + print(f"\n{_BOLD}Summary:{_RESET} ", end="") + parts = [] + if error_count: + parts.append(f"{_RED}{error_count} error{'s' if error_count != 1 else ''}{_RESET}") + if warning_count: + parts.append(f"{_YELLOW}{warning_count} warning{'s' if warning_count != 1 else ''}{_RESET}") + if not parts: + parts.append(f"{_GREEN}All checks passed{_RESET}") + print(", ".join(parts)) + + # Exit code + if error_count > 0: + return 1 + if warning_count > 0 and strict: + return 1 + return 0 + + +def _cross_reference_checks( + loaded: Dict[str, Dict[str, Any]], + result: ValidationResult, + business_roles: Set[str], + tech_roles: Set[str], + warehouses: Set[str], + resource_monitors: Set[str], +) -> None: + """Run cross-file reference integrity checks.""" + + # 1. Users' default_role -> valid roles + users = loaded.get("user.yaml", {}) + bad_default_roles = [] + for user_name, user_cfg in users.items(): + if not isinstance(user_cfg, dict): + continue + default_role = user_cfg.get("default_role") + if not default_role: + continue + role_upper = str(default_role).upper() + is_system = role_upper in SYSTEM_ROLES + is_business_raw = role_upper in business_roles + is_business_suffixed = role_upper.replace(BUSINESS_ROLE_SUFFIX, "") in business_roles + if not (is_system or is_business_raw or is_business_suffixed): + bad_default_roles.append((user_name, default_role)) + + if bad_default_roles: + for user_name, role in bad_default_roles: + result.error(f"User {user_name}: default_role '{role}' not defined as a business or system role") + else: + result.ok("All user default_roles reference valid roles") + + # 2. Business roles' tech_roles -> valid tech roles + br_data = loaded.get("business_role.yaml", {}) + bad_tech_refs = [] + for role_name, role_cfg in br_data.items(): + if not isinstance(role_cfg, dict): + continue + for tr in role_cfg.get("tech_roles", []): + if str(tr).upper() not in tech_roles: + bad_tech_refs.append((role_name, tr)) + + if bad_tech_refs: + for role_name, tr in bad_tech_refs: + result.error(f"Business role {role_name}: tech_role '{tr}' not defined in tech_role.yaml") + else: + result.ok("All business role tech_roles exist") + + # 3. Business roles' warehouse_usage -> valid warehouses + bad_wh_refs = [] + for role_name, role_cfg in br_data.items(): + if not isinstance(role_cfg, dict): + continue + for wh in role_cfg.get("warehouse_usage", []): + if str(wh).upper() not in warehouses: + bad_wh_refs.append((role_name, wh)) + + if bad_wh_refs: + for role_name, wh in bad_wh_refs: + result.error(f"Business role {role_name}: warehouse '{wh}' not defined in warehouse.yaml") + else: + result.ok("All business role warehouse_usage references exist") + + # 4. Warehouses' resource_monitor -> valid monitors + wh_data = loaded.get("warehouse.yaml", {}) + bad_monitor_refs = [] + for wh_name, wh_cfg in wh_data.items(): + if not isinstance(wh_cfg, dict): + continue + monitor = wh_cfg.get("resource_monitor") + if monitor and str(monitor).upper() not in resource_monitors: + bad_monitor_refs.append((wh_name, monitor)) + + if bad_monitor_refs: + for wh_name, monitor in bad_monitor_refs: + result.error(f"Warehouse {wh_name}: resource_monitor '{monitor}' not defined in resource_monitor.yaml") + else: + result.ok("All warehouse resource_monitors reference valid monitors") + + # 5. Tech roles' WAREHOUSE:USAGE -> valid warehouses + tr_data = loaded.get("tech_role.yaml", {}) + bad_wh_grants = [] + for role_name, role_cfg in tr_data.items(): + if not isinstance(role_cfg, dict): + continue + for section_name in ("grants", "future_grants"): + section = role_cfg.get(section_name, {}) + if not isinstance(section, dict): + continue + for grant_key, targets in section.items(): + parts = str(grant_key).split(":", 1) + if len(parts) == 2 and parts[0].upper() == "WAREHOUSE" and isinstance(targets, list): + for wh in targets: + if str(wh).upper() not in warehouses: + bad_wh_grants.append((role_name, wh)) + + if bad_wh_grants: + for role_name, wh in bad_wh_grants: + result.error(f"Tech role {role_name}: warehouse '{wh}' in grants not defined in warehouse.yaml") + else: + result.ok("All tech role warehouse grants reference valid warehouses") + + +# --------------------------------------------------------------------------- +# CLI entry point +# --------------------------------------------------------------------------- + + +def main() -> None: + """Main entry point for the validate-config command.""" + parser = argparse.ArgumentParser( + description="Validate SnowDDL YAML configuration files before deployment.", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=( + "Examples:\n" + " uv run validate-config # Validate all configs\n" + " uv run validate-config --strict # Fail on warnings too\n" + " uv run validate-config --quiet # Only show errors\n" + " uv run validate-config snowddl/user.yaml # Validate specific file\n" + ), + ) + parser.add_argument( + "files", + nargs="*", + help="Optional file path(s) to validate (default: all configs in snowddl/)", + ) + parser.add_argument( + "--strict", + action="store_true", + help="Treat warnings as errors (exit code 1 for warnings)", + ) + parser.add_argument( + "--quiet", + action="store_true", + help="Only show errors, suppress informational and warning output", + ) + parser.add_argument( + "--config-dir", + type=str, + default=None, + help="Path to the snowddl config directory (auto-detected by default)", + ) + + args = parser.parse_args() + + # Determine config directory + if args.config_dir: + config_dir = Path(args.config_dir) + else: + project_root = _find_project_root() + config_dir = project_root / "snowddl" + + target_files = args.files if args.files else None + + exit_code = run_validation(config_dir, target_files, args.strict, args.quiet) + sys.exit(exit_code) + + +if __name__ == "__main__": + main() diff --git a/snowddl/user.yaml b/snowddl/user.yaml index 031419d..c5e1ce8 100644 --- a/snowddl/user.yaml +++ b/snowddl/user.yaml @@ -24,7 +24,7 @@ JOHN_ANALYST: email: john.analyst@example.com first_name: John last_name: Analyst - default_role: ANALYST__B_ROLE + default_role: COMPANY_USERS__B_ROLE comment: Data analyst # ============================================================================= @@ -46,7 +46,7 @@ DLT_SERVICE: MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA0example... ...replace with your actual RSA public key... AQAB - default_role: DLT_INGESTION_ROLE__B_ROLE + default_role: DLT_STRIPE_ROLE__B_ROLE comment: Data ingestion service account # ============================================================================= diff --git a/src/management_cli.py b/src/management_cli.py index 143a897..4638fac 100644 --- a/src/management_cli.py +++ b/src/management_cli.py @@ -159,6 +159,16 @@ def update_user_password(): update_user_password() +# === CONFIGURATION VALIDATION === + + +def validate_config(): + """Validate SnowDDL YAML configuration files before deployment.""" + from validate_config import main + + main() + + # === STREAMLIT TESTING COMMANDS === @@ -226,6 +236,13 @@ def github_to_snowddl(): main() +def generate_terraform(): + """Generate Terraform HCL files from SnowDDL YAML configurations.""" + from generate_terraform import main + + main() + + if __name__ == "__main__": print("Use 'uv run ' where command includes:") print(" Core: warehouses, costs, security, backup, users") diff --git a/terraform-snowflake-boilerplate/README.md b/terraform-snowflake-boilerplate/README.md new file mode 100644 index 0000000..07d9546 --- /dev/null +++ b/terraform-snowflake-boilerplate/README.md @@ -0,0 +1,39 @@ +# Terraform Snowflake Boilerplate + +Auto-generated Terraform HCL files from SnowDDL YAML configurations. + +## Usage + +These files were generated by: + +```bash +uv run generate-terraform --output ./terraform-snowflake-boilerplate/ +``` + +To regenerate after YAML changes: + +```bash +uv run generate-terraform --output ./terraform-snowflake-boilerplate/ +``` + +## Files + +| File | Contents | +|------|----------| +| `main.tf` | Provider configuration | +| `users.tf` | `snowflake_user` resources | +| `warehouses.tf` | `snowflake_warehouse` resources | +| `roles.tf` | `snowflake_account_role` resources (business + tech) | +| `grants.tf` | `snowflake_grant_*` resources | +| `policies.tf` | Network, authentication, password, session policies | +| `databases.tf` | `snowflake_database` resources | +| `resource_monitors.tf` | `snowflake_resource_monitor` resources | + +## Import Support + +Each resource includes an `import {}` block so `terraform plan` can import existing Snowflake objects rather than attempting to recreate them. + +## Important + +These files are a starting point. Review and customize before running `terraform apply`. +The Snowflake provider requires authentication configuration not included here. diff --git a/terraform-snowflake-boilerplate/databases.tf b/terraform-snowflake-boilerplate/databases.tf new file mode 100644 index 0000000..eb00b9f --- /dev/null +++ b/terraform-snowflake-boilerplate/databases.tf @@ -0,0 +1,34 @@ +# Generated by SnowTower - DO NOT EDIT MANUALLY +# Source: snowddl/ YAML configurations + +# Database resources + +import { + to = snowflake_database.dev_example + id = "DEV_EXAMPLE" +} + +resource "snowflake_database" "dev_example" { + name = "DEV_EXAMPLE" + comment = "Sandbox database DEV_EXAMPLE" +} + +import { + to = snowflake_database.lightdash + id = "LIGHTDASH" +} + +resource "snowflake_database" "lightdash" { + name = "LIGHTDASH" + comment = "LightDash BI platform database for analytics transformations and dashboard content" +} + +import { + to = snowflake_database.omni + id = "OMNI" +} + +resource "snowflake_database" "omni" { + name = "OMNI" + comment = "Omni BI platform database for analytics transformations and custom reporting" +} diff --git a/terraform-snowflake-boilerplate/grants.tf b/terraform-snowflake-boilerplate/grants.tf new file mode 100644 index 0000000..78f75c8 --- /dev/null +++ b/terraform-snowflake-boilerplate/grants.tf @@ -0,0 +1,1791 @@ +# Generated by SnowTower - DO NOT EDIT MANUALLY +# Source: snowddl/ YAML configurations + +# Grant resources + +resource "snowflake_grant_account_role" "admin_role_b_role_inherits_streamlit_viewer_t_role" { + role_name = "STREAMLIT_VIEWER__T_ROLE" + parent_role_name = "ADMIN_ROLE__B_ROLE" +} + +resource "snowflake_grant_account_role" "admin_role_b_role_inherits_dlt_loader_role_t_role" { + role_name = "DLT_LOADER_ROLE__T_ROLE" + parent_role_name = "ADMIN_ROLE__B_ROLE" +} + +resource "snowflake_grant_account_role" "admin_role_b_role_inherits_snowtower_users_t_role" { + role_name = "SNOWTOWER_USERS__T_ROLE" + parent_role_name = "ADMIN_ROLE__B_ROLE" +} + +resource "snowflake_grant_account_role" "admin_role_b_role_inherits_stripe_t_role" { + role_name = "STRIPE__T_ROLE" + parent_role_name = "ADMIN_ROLE__B_ROLE" +} + +resource "snowflake_grant_account_role" "admin_role_b_role_inherits_company_users_t_role" { + role_name = "COMPANY_USERS__T_ROLE" + parent_role_name = "ADMIN_ROLE__B_ROLE" +} + +resource "snowflake_grant_privileges_to_account_role" "admin_role_b_role_wh_admin" { + account_role_name = "ADMIN_ROLE__B_ROLE" + privileges = ["USAGE"] + on_account_object { + object_type = "WAREHOUSE" + object_name = "ADMIN" + } +} + +resource "snowflake_grant_privileges_to_account_role" "admin_role_b_role_wh_main_warehouse" { + account_role_name = "ADMIN_ROLE__B_ROLE" + privileges = ["USAGE"] + on_account_object { + object_type = "WAREHOUSE" + object_name = "MAIN_WAREHOUSE" + } +} + +resource "snowflake_grant_account_role" "ai_ml_role_b_role_inherits_fabi_ai_t_role" { + role_name = "FABI_AI__T_ROLE" + parent_role_name = "AI_ML_ROLE__B_ROLE" +} + +resource "snowflake_grant_account_role" "ai_ml_role_b_role_inherits_copilot_access_role_t_role" { + role_name = "COPILOT_ACCESS_ROLE__T_ROLE" + parent_role_name = "AI_ML_ROLE__B_ROLE" +} + +resource "snowflake_grant_privileges_to_account_role" "ai_ml_role_b_role_wh_fabi_ai_warehouse" { + account_role_name = "AI_ML_ROLE__B_ROLE" + privileges = ["USAGE"] + on_account_object { + object_type = "WAREHOUSE" + object_name = "FABI_AI_WAREHOUSE" + } +} + +resource "snowflake_grant_account_role" "bi_developer_role_b_role_inherits_streamlit_viewer_t_role" { + role_name = "STREAMLIT_VIEWER__T_ROLE" + parent_role_name = "BI_DEVELOPER_ROLE__B_ROLE" +} + +resource "snowflake_grant_account_role" "bi_developer_role_b_role_inherits_dbt_stripe_role_t_role" { + role_name = "DBT_STRIPE_ROLE__T_ROLE" + parent_role_name = "BI_DEVELOPER_ROLE__B_ROLE" +} + +resource "snowflake_grant_account_role" "bi_developer_role_b_role_inherits_bi_writer_tech_role_t_role" { + role_name = "BI_WRITER_TECH_ROLE__T_ROLE" + parent_role_name = "BI_DEVELOPER_ROLE__B_ROLE" +} + +resource "snowflake_grant_account_role" "bi_developer_role_b_role_inherits_sigma_role_t_role" { + role_name = "SIGMA_ROLE__T_ROLE" + parent_role_name = "BI_DEVELOPER_ROLE__B_ROLE" +} + +resource "snowflake_grant_privileges_to_account_role" "bi_developer_role_b_role_wh_transforming" { + account_role_name = "BI_DEVELOPER_ROLE__B_ROLE" + privileges = ["USAGE"] + on_account_object { + object_type = "WAREHOUSE" + object_name = "TRANSFORMING" + } +} + +resource "snowflake_grant_privileges_to_account_role" "bi_developer_role_b_role_wh_analytics_tool" { + account_role_name = "BI_DEVELOPER_ROLE__B_ROLE" + privileges = ["USAGE"] + on_account_object { + object_type = "WAREHOUSE" + object_name = "ANALYTICS_TOOL" + } +} + +resource "snowflake_grant_privileges_to_account_role" "bi_developer_role_b_role_wh_main_warehouse" { + account_role_name = "BI_DEVELOPER_ROLE__B_ROLE" + privileges = ["USAGE"] + on_account_object { + object_type = "WAREHOUSE" + object_name = "MAIN_WAREHOUSE" + } +} + +resource "snowflake_grant_account_role" "company_users_b_role_inherits_streamlit_viewer_t_role" { + role_name = "STREAMLIT_VIEWER__T_ROLE" + parent_role_name = "COMPANY_USERS__B_ROLE" +} + +resource "snowflake_grant_account_role" "company_users_b_role_inherits_dlt_loader_role_t_role" { + role_name = "DLT_LOADER_ROLE__T_ROLE" + parent_role_name = "COMPANY_USERS__B_ROLE" +} + +resource "snowflake_grant_account_role" "company_users_b_role_inherits_stripe_t_role" { + role_name = "STRIPE__T_ROLE" + parent_role_name = "COMPANY_USERS__B_ROLE" +} + +resource "snowflake_grant_account_role" "company_users_b_role_inherits_company_users_t_role" { + role_name = "COMPANY_USERS__T_ROLE" + parent_role_name = "COMPANY_USERS__B_ROLE" +} + +resource "snowflake_grant_account_role" "company_users_b_role_inherits_dbt_stripe_role_t_role" { + role_name = "DBT_STRIPE_ROLE__T_ROLE" + parent_role_name = "COMPANY_USERS__B_ROLE" +} + +resource "snowflake_grant_privileges_to_account_role" "company_users_b_role_wh_main_warehouse" { + account_role_name = "COMPANY_USERS__B_ROLE" + privileges = ["USAGE"] + on_account_object { + object_type = "WAREHOUSE" + object_name = "MAIN_WAREHOUSE" + } +} + +resource "snowflake_grant_privileges_to_account_role" "company_users_b_role_wh_dlt" { + account_role_name = "COMPANY_USERS__B_ROLE" + privileges = ["USAGE"] + on_account_object { + object_type = "WAREHOUSE" + object_name = "DLT" + } +} + +resource "snowflake_grant_privileges_to_account_role" "company_users_b_role_wh_stripe" { + account_role_name = "COMPANY_USERS__B_ROLE" + privileges = ["USAGE"] + on_account_object { + object_type = "WAREHOUSE" + object_name = "STRIPE" + } +} + +resource "snowflake_grant_privileges_to_account_role" "company_users_b_role_wh_transforming" { + account_role_name = "COMPANY_USERS__B_ROLE" + privileges = ["USAGE"] + on_account_object { + object_type = "WAREHOUSE" + object_name = "TRANSFORMING" + } +} + +resource "snowflake_grant_privileges_to_account_role" "company_users_b_role_wh_dev_wh" { + account_role_name = "COMPANY_USERS__B_ROLE" + privileges = ["USAGE"] + on_account_object { + object_type = "WAREHOUSE" + object_name = "DEV_WH" + } +} + +resource "snowflake_grant_account_role" "data_integration_role_b_role_inherits_estuary_t_role" { + role_name = "ESTUARY__T_ROLE" + parent_role_name = "DATA_INTEGRATION_ROLE__B_ROLE" +} + +resource "snowflake_grant_account_role" "data_integration_role_b_role_inherits_fivetran_t_role" { + role_name = "FIVETRAN__T_ROLE" + parent_role_name = "DATA_INTEGRATION_ROLE__B_ROLE" +} + +resource "snowflake_grant_account_role" "data_integration_role_b_role_inherits_fivetran_role_t_role" { + role_name = "FIVETRAN_ROLE__T_ROLE" + parent_role_name = "DATA_INTEGRATION_ROLE__B_ROLE" +} + +resource "snowflake_grant_account_role" "data_integration_role_b_role_inherits_matillion_role_t_role" { + role_name = "MATILLION_ROLE__T_ROLE" + parent_role_name = "DATA_INTEGRATION_ROLE__B_ROLE" +} + +resource "snowflake_grant_account_role" "data_integration_role_b_role_inherits_dlt_loader_role_t_role" { + role_name = "DLT_LOADER_ROLE__T_ROLE" + parent_role_name = "DATA_INTEGRATION_ROLE__B_ROLE" +} + +resource "snowflake_grant_privileges_to_account_role" "data_integration_role_b_role_wh_estuary" { + account_role_name = "DATA_INTEGRATION_ROLE__B_ROLE" + privileges = ["USAGE"] + on_account_object { + object_type = "WAREHOUSE" + object_name = "ESTUARY" + } +} + +resource "snowflake_grant_privileges_to_account_role" "data_integration_role_b_role_wh_fivetran" { + account_role_name = "DATA_INTEGRATION_ROLE__B_ROLE" + privileges = ["USAGE"] + on_account_object { + object_type = "WAREHOUSE" + object_name = "FIVETRAN" + } +} + +resource "snowflake_grant_privileges_to_account_role" "data_integration_role_b_role_wh_transforming" { + account_role_name = "DATA_INTEGRATION_ROLE__B_ROLE" + privileges = ["USAGE"] + on_account_object { + object_type = "WAREHOUSE" + object_name = "TRANSFORMING" + } +} + +resource "snowflake_grant_privileges_to_account_role" "data_integration_role_b_role_wh_stripe" { + account_role_name = "DATA_INTEGRATION_ROLE__B_ROLE" + privileges = ["USAGE"] + on_account_object { + object_type = "WAREHOUSE" + object_name = "STRIPE" + } +} + +resource "snowflake_grant_account_role" "dbt_analytics_role_b_role_inherits_dbt_stripe_role_t_role" { + role_name = "DBT_STRIPE_ROLE__T_ROLE" + parent_role_name = "DBT_ANALYTICS_ROLE__B_ROLE" +} + +resource "snowflake_grant_account_role" "dbt_analytics_role_b_role_inherits_dbt_training_t_role" { + role_name = "DBT_TRAINING__T_ROLE" + parent_role_name = "DBT_ANALYTICS_ROLE__B_ROLE" +} + +resource "snowflake_grant_privileges_to_account_role" "dbt_analytics_role_b_role_wh_main_warehouse" { + account_role_name = "DBT_ANALYTICS_ROLE__B_ROLE" + privileges = ["USAGE"] + on_account_object { + object_type = "WAREHOUSE" + object_name = "MAIN_WAREHOUSE" + } +} + +resource "snowflake_grant_privileges_to_account_role" "dbt_analytics_role_b_role_wh_transforming" { + account_role_name = "DBT_ANALYTICS_ROLE__B_ROLE" + privileges = ["USAGE"] + on_account_object { + object_type = "WAREHOUSE" + object_name = "TRANSFORMING" + } +} + +resource "snowflake_grant_account_role" "dbt_analytics_role_b_role_owner_proj_stripe_proj_stripe" { + role_name = "PROJ_STRIPE__PROJ_STRIPE__OWNER__S_ROLE" + parent_role_name = "DBT_ANALYTICS_ROLE__B_ROLE" +} + +resource "snowflake_grant_account_role" "dlt_stripe_role_b_role_inherits_dlt_stripe_tech_role_t_role" { + role_name = "DLT_STRIPE_TECH_ROLE__T_ROLE" + parent_role_name = "DLT_STRIPE_ROLE__B_ROLE" +} + +resource "snowflake_grant_privileges_to_account_role" "dlt_stripe_role_b_role_wh_dlt" { + account_role_name = "DLT_STRIPE_ROLE__B_ROLE" + privileges = ["USAGE"] + on_account_object { + object_type = "WAREHOUSE" + object_name = "DLT" + } +} + +resource "snowflake_grant_account_role" "dlt_stripe_role_b_role_owner_source_stripe_stripe_why" { + role_name = "SOURCE_STRIPE__STRIPE_WHY__OWNER__S_ROLE" + parent_role_name = "DLT_STRIPE_ROLE__B_ROLE" +} + +resource "snowflake_grant_account_role" "infrastructure_automation_role_b_role_inherits_omni_infrastructure_automation_t_role" { + role_name = "OMNI_INFRASTRUCTURE_AUTOMATION__T_ROLE" + parent_role_name = "INFRASTRUCTURE_AUTOMATION_ROLE__B_ROLE" +} + +resource "snowflake_grant_account_role" "infrastructure_automation_role_b_role_inherits_tobiko_cloud_t_role" { + role_name = "TOBIKO_CLOUD__T_ROLE" + parent_role_name = "INFRASTRUCTURE_AUTOMATION_ROLE__B_ROLE" +} + +resource "snowflake_grant_privileges_to_account_role" "infrastructure_automation_role_b_role_wh_analytics_tool" { + account_role_name = "INFRASTRUCTURE_AUTOMATION_ROLE__B_ROLE" + privileges = ["USAGE"] + on_account_object { + object_type = "WAREHOUSE" + object_name = "ANALYTICS_TOOL" + } +} + +resource "snowflake_grant_privileges_to_account_role" "infrastructure_automation_role_b_role_wh_admin" { + account_role_name = "INFRASTRUCTURE_AUTOMATION_ROLE__B_ROLE" + privileges = ["USAGE"] + on_account_object { + object_type = "WAREHOUSE" + object_name = "ADMIN" + } +} + +resource "snowflake_grant_account_role" "lightdash_business_role_b_role_inherits_lightdash_tech_role_t_role" { + role_name = "LIGHTDASH_TECH_ROLE__T_ROLE" + parent_role_name = "LIGHTDASH_BUSINESS_ROLE__B_ROLE" +} + +resource "snowflake_grant_privileges_to_account_role" "lightdash_business_role_b_role_wh_bi_tool" { + account_role_name = "LIGHTDASH_BUSINESS_ROLE__B_ROLE" + privileges = ["USAGE"] + on_account_object { + object_type = "WAREHOUSE" + object_name = "BI_TOOL" + } +} + +resource "snowflake_grant_account_role" "omni_business_role_b_role_inherits_omni_tech_role_t_role" { + role_name = "OMNI_TECH_ROLE__T_ROLE" + parent_role_name = "OMNI_BUSINESS_ROLE__B_ROLE" +} + +resource "snowflake_grant_privileges_to_account_role" "omni_business_role_b_role_wh_analytics_tool" { + account_role_name = "OMNI_BUSINESS_ROLE__B_ROLE" + privileges = ["USAGE"] + on_account_object { + object_type = "WAREHOUSE" + object_name = "ANALYTICS_TOOL" + } +} + +resource "snowflake_grant_account_role" "recce_business_role_b_role_inherits_recce_tech_role_t_role" { + role_name = "RECCE_TECH_ROLE__T_ROLE" + parent_role_name = "RECCE_BUSINESS_ROLE__B_ROLE" +} + +resource "snowflake_grant_privileges_to_account_role" "recce_business_role_b_role_wh_recce" { + account_role_name = "RECCE_BUSINESS_ROLE__B_ROLE" + privileges = ["USAGE"] + on_account_object { + object_type = "WAREHOUSE" + object_name = "RECCE" + } +} + +resource "snowflake_grant_account_role" "service_accounts_role_b_role_inherits_tobiko_cloud_t_role" { + role_name = "TOBIKO_CLOUD__T_ROLE" + parent_role_name = "SERVICE_ACCOUNTS_ROLE__B_ROLE" +} + +resource "snowflake_grant_privileges_to_account_role" "service_accounts_role_b_role_wh_admin" { + account_role_name = "SERVICE_ACCOUNTS_ROLE__B_ROLE" + privileges = ["USAGE"] + on_account_object { + object_type = "WAREHOUSE" + object_name = "ADMIN" + } +} + +resource "snowflake_grant_account_role" "streamlit_apps_role_b_role_inherits_streamlit_role_t_role" { + role_name = "STREAMLIT_ROLE__T_ROLE" + parent_role_name = "STREAMLIT_APPS_ROLE__B_ROLE" +} + +resource "snowflake_grant_account_role" "streamlit_apps_role_b_role_inherits_streamlit_towerapp_role_t_role" { + role_name = "STREAMLIT_TOWERAPP_ROLE__T_ROLE" + parent_role_name = "STREAMLIT_APPS_ROLE__B_ROLE" +} + +resource "snowflake_grant_account_role" "streamlit_apps_role_b_role_inherits_streamlit_viewer_t_role" { + role_name = "STREAMLIT_VIEWER__T_ROLE" + parent_role_name = "STREAMLIT_APPS_ROLE__B_ROLE" +} + +resource "snowflake_grant_privileges_to_account_role" "streamlit_apps_role_b_role_wh_streamlit_viewer_wh" { + account_role_name = "STREAMLIT_APPS_ROLE__B_ROLE" + privileges = ["USAGE"] + on_account_object { + object_type = "WAREHOUSE" + object_name = "STREAMLIT_VIEWER_WH" + } +} + +resource "snowflake_grant_privileges_to_account_role" "streamlit_apps_role_b_role_wh_main_warehouse" { + account_role_name = "STREAMLIT_APPS_ROLE__B_ROLE" + privileges = ["USAGE"] + on_account_object { + object_type = "WAREHOUSE" + object_name = "MAIN_WAREHOUSE" + } +} + +resource "snowflake_grant_account_role" "training_role_b_role_inherits_dbt_training_t_role" { + role_name = "DBT_TRAINING__T_ROLE" + parent_role_name = "TRAINING_ROLE__B_ROLE" +} + +resource "snowflake_grant_privileges_to_account_role" "training_role_b_role_wh_transforming_user" { + account_role_name = "TRAINING_ROLE__B_ROLE" + privileges = ["USAGE"] + on_account_object { + object_type = "WAREHOUSE" + object_name = "TRANSFORMING_USER" + } +} + +resource "snowflake_grant_privileges_to_account_role" "training_role_b_role_wh_transforming" { + account_role_name = "TRAINING_ROLE__B_ROLE" + privileges = ["USAGE"] + on_account_object { + object_type = "WAREHOUSE" + object_name = "TRANSFORMING" + } +} + +resource "snowflake_grant_privileges_to_account_role" "bi_writer_tech_role_t_role_grant_database_proj_stripe_usage" { + account_role_name = "BI_WRITER_TECH_ROLE__T_ROLE" + privileges = ["USAGE"] + on_account_object { + object_type = "DATABASE" + object_name = "PROJ_STRIPE" + } +} + +resource "snowflake_grant_privileges_to_account_role" "bi_writer_tech_role_t_role_grant_database_source_stripe_usage" { + account_role_name = "BI_WRITER_TECH_ROLE__T_ROLE" + privileges = ["USAGE"] + on_account_object { + object_type = "DATABASE" + object_name = "SOURCE_STRIPE" + } +} + +resource "snowflake_grant_privileges_to_account_role" "bi_writer_tech_role_t_role_grant_database_analytics_tool_usage_create_schema" { + account_role_name = "BI_WRITER_TECH_ROLE__T_ROLE" + privileges = ["USAGE", "CREATE SCHEMA"] + on_account_object { + object_type = "DATABASE" + object_name = "ANALYTICS_TOOL" + } +} + +resource "snowflake_grant_privileges_to_account_role" "bi_writer_tech_role_t_role_grant_schema_source_stripe_stripe_why_usage" { + account_role_name = "BI_WRITER_TECH_ROLE__T_ROLE" + privileges = ["USAGE"] + on_schema { + schema_name = "\"SOURCE_STRIPE.STRIPE_WHY\"" + } +} + +resource "snowflake_grant_privileges_to_account_role" "bi_writer_tech_role_t_role_grant_warehouse_transforming_usage" { + account_role_name = "BI_WRITER_TECH_ROLE__T_ROLE" + privileges = ["USAGE"] + on_account_object { + object_type = "WAREHOUSE" + object_name = "TRANSFORMING" + } +} + +resource "snowflake_grant_privileges_to_account_role" "bi_writer_tech_role_t_role_grant_warehouse_analytics_tool_usage" { + account_role_name = "BI_WRITER_TECH_ROLE__T_ROLE" + privileges = ["USAGE"] + on_account_object { + object_type = "WAREHOUSE" + object_name = "ANALYTICS_TOOL" + } +} + +resource "snowflake_grant_privileges_to_account_role" "bi_writer_tech_role_t_role_future_table_proj_stripe_select" { + account_role_name = "BI_WRITER_TECH_ROLE__T_ROLE" + privileges = ["SELECT"] + on_schema_object { + object_type_plural = "TABLES" + in_database = "PROJ_STRIPE" + } + all_privileges = false + with_grant_option = false +} + +resource "snowflake_grant_privileges_to_account_role" "bi_writer_tech_role_t_role_future_table_source_stripe_select" { + account_role_name = "BI_WRITER_TECH_ROLE__T_ROLE" + privileges = ["SELECT"] + on_schema_object { + object_type_plural = "TABLES" + in_database = "SOURCE_STRIPE" + } + all_privileges = false + with_grant_option = false +} + +resource "snowflake_grant_privileges_to_account_role" "bi_writer_tech_role_t_role_future_table_analytics_tool_select_insert_update_delete_truncate" { + account_role_name = "BI_WRITER_TECH_ROLE__T_ROLE" + privileges = ["SELECT", "INSERT", "UPDATE", "DELETE", "TRUNCATE"] + on_schema_object { + object_type_plural = "TABLES" + in_database = "ANALYTICS_TOOL" + } + all_privileges = false + with_grant_option = false +} + +resource "snowflake_grant_privileges_to_account_role" "bi_writer_tech_role_t_role_future_view_analytics_tool_select" { + account_role_name = "BI_WRITER_TECH_ROLE__T_ROLE" + privileges = ["SELECT"] + on_schema_object { + object_type_plural = "VIEWS" + in_database = "ANALYTICS_TOOL" + } + all_privileges = false + with_grant_option = false +} + +resource "snowflake_grant_privileges_to_account_role" "company_users_t_role_grant_database_dev_alice_usage_create_schema" { + account_role_name = "COMPANY_USERS__T_ROLE" + privileges = ["USAGE", "CREATE SCHEMA"] + on_account_object { + object_type = "DATABASE" + object_name = "DEV_ALICE" + } +} + +resource "snowflake_grant_privileges_to_account_role" "company_users_t_role_grant_database_dev_carol_usage_create_schema" { + account_role_name = "COMPANY_USERS__T_ROLE" + privileges = ["USAGE", "CREATE SCHEMA"] + on_account_object { + object_type = "DATABASE" + object_name = "DEV_CAROL" + } +} + +resource "snowflake_grant_privileges_to_account_role" "company_users_t_role_grant_database_dev_dave_usage_create_schema" { + account_role_name = "COMPANY_USERS__T_ROLE" + privileges = ["USAGE", "CREATE SCHEMA"] + on_account_object { + object_type = "DATABASE" + object_name = "DEV_DAVE" + } +} + +resource "snowflake_grant_privileges_to_account_role" "company_users_t_role_grant_database_dev_eve_usage_create_schema" { + account_role_name = "COMPANY_USERS__T_ROLE" + privileges = ["USAGE", "CREATE SCHEMA"] + on_account_object { + object_type = "DATABASE" + object_name = "DEV_EVE" + } +} + +resource "snowflake_grant_privileges_to_account_role" "company_users_t_role_grant_database_dev_grace_usage_create_schema" { + account_role_name = "COMPANY_USERS__T_ROLE" + privileges = ["USAGE", "CREATE SCHEMA"] + on_account_object { + object_type = "DATABASE" + object_name = "DEV_GRACE" + } +} + +resource "snowflake_grant_privileges_to_account_role" "company_users_t_role_grant_database_dev_bob_usage_create_schema" { + account_role_name = "COMPANY_USERS__T_ROLE" + privileges = ["USAGE", "CREATE SCHEMA"] + on_account_object { + object_type = "DATABASE" + object_name = "DEV_BOB" + } +} + +resource "snowflake_grant_privileges_to_account_role" "company_users_t_role_grant_database_dev_frank_usage_create_schema" { + account_role_name = "COMPANY_USERS__T_ROLE" + privileges = ["USAGE", "CREATE SCHEMA"] + on_account_object { + object_type = "DATABASE" + object_name = "DEV_FRANK" + } +} + +resource "snowflake_grant_privileges_to_account_role" "company_users_t_role_grant_schema_source_stripe_stripe_why_usage" { + account_role_name = "COMPANY_USERS__T_ROLE" + privileges = ["USAGE"] + on_schema { + schema_name = "\"SOURCE_STRIPE.STRIPE_WHY\"" + } +} + +resource "snowflake_grant_privileges_to_account_role" "company_users_t_role_grant_schema_dev_alice_stripe_why_usage_modify_monitor_create_table_create_view_create_function_create_procedure_create_file_format_create_stage_create_sequence" { + account_role_name = "COMPANY_USERS__T_ROLE" + privileges = ["USAGE", "MODIFY", "MONITOR", "CREATE TABLE", "CREATE VIEW", "CREATE FUNCTION", "CREATE PROCEDURE", "CREATE FILE FORMAT", "CREATE STAGE", "CREATE SEQUENCE"] + on_schema { + schema_name = "\"DEV_ALICE.STRIPE_WHY\"" + } +} + +resource "snowflake_grant_privileges_to_account_role" "company_users_t_role_grant_warehouse_main_warehouse_usage" { + account_role_name = "COMPANY_USERS__T_ROLE" + privileges = ["USAGE"] + on_account_object { + object_type = "WAREHOUSE" + object_name = "MAIN_WAREHOUSE" + } +} + +resource "snowflake_grant_privileges_to_account_role" "company_users_t_role_future_stage_dev_alice_usage_read_write" { + account_role_name = "COMPANY_USERS__T_ROLE" + privileges = ["USAGE", "READ", "WRITE"] + on_schema_object { + object_type_plural = "STAGES" + in_database = "DEV_ALICE" + } + all_privileges = false + with_grant_option = false +} + +resource "snowflake_grant_privileges_to_account_role" "company_users_t_role_future_stage_dev_carol_usage_read_write" { + account_role_name = "COMPANY_USERS__T_ROLE" + privileges = ["USAGE", "READ", "WRITE"] + on_schema_object { + object_type_plural = "STAGES" + in_database = "DEV_CAROL" + } + all_privileges = false + with_grant_option = false +} + +resource "snowflake_grant_privileges_to_account_role" "company_users_t_role_future_stage_dev_dave_usage_read_write" { + account_role_name = "COMPANY_USERS__T_ROLE" + privileges = ["USAGE", "READ", "WRITE"] + on_schema_object { + object_type_plural = "STAGES" + in_database = "DEV_DAVE" + } + all_privileges = false + with_grant_option = false +} + +resource "snowflake_grant_privileges_to_account_role" "company_users_t_role_future_stage_dev_eve_usage_read_write" { + account_role_name = "COMPANY_USERS__T_ROLE" + privileges = ["USAGE", "READ", "WRITE"] + on_schema_object { + object_type_plural = "STAGES" + in_database = "DEV_EVE" + } + all_privileges = false + with_grant_option = false +} + +resource "snowflake_grant_privileges_to_account_role" "company_users_t_role_future_stage_dev_grace_usage_read_write" { + account_role_name = "COMPANY_USERS__T_ROLE" + privileges = ["USAGE", "READ", "WRITE"] + on_schema_object { + object_type_plural = "STAGES" + in_database = "DEV_GRACE" + } + all_privileges = false + with_grant_option = false +} + +resource "snowflake_grant_privileges_to_account_role" "company_users_t_role_future_stage_dev_bob_usage_read_write" { + account_role_name = "COMPANY_USERS__T_ROLE" + privileges = ["USAGE", "READ", "WRITE"] + on_schema_object { + object_type_plural = "STAGES" + in_database = "DEV_BOB" + } + all_privileges = false + with_grant_option = false +} + +resource "snowflake_grant_privileges_to_account_role" "company_users_t_role_future_stage_dev_frank_usage_read_write" { + account_role_name = "COMPANY_USERS__T_ROLE" + privileges = ["USAGE", "READ", "WRITE"] + on_schema_object { + object_type_plural = "STAGES" + in_database = "DEV_FRANK" + } + all_privileges = false + with_grant_option = false +} + +resource "snowflake_grant_privileges_to_account_role" "company_users_t_role_future_table_dev_alice_select_insert_update_delete_truncate" { + account_role_name = "COMPANY_USERS__T_ROLE" + privileges = ["SELECT", "INSERT", "UPDATE", "DELETE", "TRUNCATE"] + on_schema_object { + object_type_plural = "TABLES" + in_database = "DEV_ALICE" + } + all_privileges = false + with_grant_option = false +} + +resource "snowflake_grant_privileges_to_account_role" "company_users_t_role_future_table_dev_carol_select_insert_update_delete_truncate" { + account_role_name = "COMPANY_USERS__T_ROLE" + privileges = ["SELECT", "INSERT", "UPDATE", "DELETE", "TRUNCATE"] + on_schema_object { + object_type_plural = "TABLES" + in_database = "DEV_CAROL" + } + all_privileges = false + with_grant_option = false +} + +resource "snowflake_grant_privileges_to_account_role" "company_users_t_role_future_table_dev_dave_select_insert_update_delete_truncate" { + account_role_name = "COMPANY_USERS__T_ROLE" + privileges = ["SELECT", "INSERT", "UPDATE", "DELETE", "TRUNCATE"] + on_schema_object { + object_type_plural = "TABLES" + in_database = "DEV_DAVE" + } + all_privileges = false + with_grant_option = false +} + +resource "snowflake_grant_privileges_to_account_role" "company_users_t_role_future_table_dev_eve_select_insert_update_delete_truncate" { + account_role_name = "COMPANY_USERS__T_ROLE" + privileges = ["SELECT", "INSERT", "UPDATE", "DELETE", "TRUNCATE"] + on_schema_object { + object_type_plural = "TABLES" + in_database = "DEV_EVE" + } + all_privileges = false + with_grant_option = false +} + +resource "snowflake_grant_privileges_to_account_role" "company_users_t_role_future_table_dev_grace_select_insert_update_delete_truncate" { + account_role_name = "COMPANY_USERS__T_ROLE" + privileges = ["SELECT", "INSERT", "UPDATE", "DELETE", "TRUNCATE"] + on_schema_object { + object_type_plural = "TABLES" + in_database = "DEV_GRACE" + } + all_privileges = false + with_grant_option = false +} + +resource "snowflake_grant_privileges_to_account_role" "company_users_t_role_future_table_dev_bob_select_insert_update_delete_truncate" { + account_role_name = "COMPANY_USERS__T_ROLE" + privileges = ["SELECT", "INSERT", "UPDATE", "DELETE", "TRUNCATE"] + on_schema_object { + object_type_plural = "TABLES" + in_database = "DEV_BOB" + } + all_privileges = false + with_grant_option = false +} + +resource "snowflake_grant_privileges_to_account_role" "company_users_t_role_future_table_dev_frank_select_insert_update_delete_truncate" { + account_role_name = "COMPANY_USERS__T_ROLE" + privileges = ["SELECT", "INSERT", "UPDATE", "DELETE", "TRUNCATE"] + on_schema_object { + object_type_plural = "TABLES" + in_database = "DEV_FRANK" + } + all_privileges = false + with_grant_option = false +} + +resource "snowflake_grant_privileges_to_account_role" "copilot_access_role_t_role_grant_warehouse_main_warehouse_usage" { + account_role_name = "COPILOT_ACCESS_ROLE__T_ROLE" + privileges = ["USAGE"] + on_account_object { + object_type = "WAREHOUSE" + object_name = "MAIN_WAREHOUSE" + } +} + +resource "snowflake_grant_privileges_to_account_role" "dbt_stripe_role_t_role_grant_database_source_stripe_usage_create_schema" { + account_role_name = "DBT_STRIPE_ROLE__T_ROLE" + privileges = ["USAGE", "CREATE SCHEMA"] + on_account_object { + object_type = "DATABASE" + object_name = "SOURCE_STRIPE" + } +} + +resource "snowflake_grant_privileges_to_account_role" "dbt_stripe_role_t_role_grant_database_proj_stripe_usage_create_schema" { + account_role_name = "DBT_STRIPE_ROLE__T_ROLE" + privileges = ["USAGE", "CREATE SCHEMA"] + on_account_object { + object_type = "DATABASE" + object_name = "PROJ_STRIPE" + } +} + +resource "snowflake_grant_privileges_to_account_role" "dbt_stripe_role_t_role_grant_database_analytics_tool_usage_create_schema" { + account_role_name = "DBT_STRIPE_ROLE__T_ROLE" + privileges = ["USAGE", "CREATE SCHEMA"] + on_account_object { + object_type = "DATABASE" + object_name = "ANALYTICS_TOOL" + } +} + +resource "snowflake_grant_privileges_to_account_role" "dbt_stripe_role_t_role_grant_schema_source_stripe_stripe_why_usage" { + account_role_name = "DBT_STRIPE_ROLE__T_ROLE" + privileges = ["USAGE"] + on_schema { + schema_name = "\"SOURCE_STRIPE.STRIPE_WHY\"" + } +} + +resource "snowflake_grant_privileges_to_account_role" "dbt_stripe_role_t_role_grant_schema_proj_stripe_proj_stripe_usage_modify_monitor_create_table_create_view_create_function_create_procedure_create_file_format_create_stage_create_sequence" { + account_role_name = "DBT_STRIPE_ROLE__T_ROLE" + privileges = ["USAGE", "MODIFY", "MONITOR", "CREATE TABLE", "CREATE VIEW", "CREATE FUNCTION", "CREATE PROCEDURE", "CREATE FILE FORMAT", "CREATE STAGE", "CREATE SEQUENCE"] + on_schema { + schema_name = "\"PROJ_STRIPE.PROJ_STRIPE\"" + } +} + +resource "snowflake_grant_privileges_to_account_role" "dbt_stripe_role_t_role_grant_warehouse_transforming_usage" { + account_role_name = "DBT_STRIPE_ROLE__T_ROLE" + privileges = ["USAGE"] + on_account_object { + object_type = "WAREHOUSE" + object_name = "TRANSFORMING" + } +} + +resource "snowflake_grant_privileges_to_account_role" "dbt_stripe_role_t_role_future_table_source_stripe_select" { + account_role_name = "DBT_STRIPE_ROLE__T_ROLE" + privileges = ["SELECT"] + on_schema_object { + object_type_plural = "TABLES" + in_database = "SOURCE_STRIPE" + } + all_privileges = false + with_grant_option = false +} + +resource "snowflake_grant_privileges_to_account_role" "dbt_stripe_role_t_role_future_table_proj_stripe_select_insert_update_delete_truncate" { + account_role_name = "DBT_STRIPE_ROLE__T_ROLE" + privileges = ["SELECT", "INSERT", "UPDATE", "DELETE", "TRUNCATE"] + on_schema_object { + object_type_plural = "TABLES" + in_database = "PROJ_STRIPE" + } + all_privileges = false + with_grant_option = false +} + +resource "snowflake_grant_privileges_to_account_role" "dbt_stripe_role_t_role_future_table_analytics_tool_select_insert_update_delete_truncate" { + account_role_name = "DBT_STRIPE_ROLE__T_ROLE" + privileges = ["SELECT", "INSERT", "UPDATE", "DELETE", "TRUNCATE"] + on_schema_object { + object_type_plural = "TABLES" + in_database = "ANALYTICS_TOOL" + } + all_privileges = false + with_grant_option = false +} + +resource "snowflake_grant_privileges_to_account_role" "dbt_stripe_role_t_role_future_view_source_stripe_select" { + account_role_name = "DBT_STRIPE_ROLE__T_ROLE" + privileges = ["SELECT"] + on_schema_object { + object_type_plural = "VIEWS" + in_database = "SOURCE_STRIPE" + } + all_privileges = false + with_grant_option = false +} + +resource "snowflake_grant_privileges_to_account_role" "dbt_stripe_role_t_role_future_view_proj_stripe_select" { + account_role_name = "DBT_STRIPE_ROLE__T_ROLE" + privileges = ["SELECT"] + on_schema_object { + object_type_plural = "VIEWS" + in_database = "PROJ_STRIPE" + } + all_privileges = false + with_grant_option = false +} + +resource "snowflake_grant_privileges_to_account_role" "dbt_stripe_role_t_role_future_view_analytics_tool_select" { + account_role_name = "DBT_STRIPE_ROLE__T_ROLE" + privileges = ["SELECT"] + on_schema_object { + object_type_plural = "VIEWS" + in_database = "ANALYTICS_TOOL" + } + all_privileges = false + with_grant_option = false +} + +resource "snowflake_grant_privileges_to_account_role" "dbt_training_t_role_grant_warehouse_transforming_usage" { + account_role_name = "DBT_TRAINING__T_ROLE" + privileges = ["USAGE"] + on_account_object { + object_type = "WAREHOUSE" + object_name = "TRANSFORMING" + } +} + +resource "snowflake_grant_privileges_to_account_role" "dlt_loader_role_t_role_grant_database_source_stripe_usage_create_schema" { + account_role_name = "DLT_LOADER_ROLE__T_ROLE" + privileges = ["USAGE", "CREATE SCHEMA"] + on_account_object { + object_type = "DATABASE" + object_name = "SOURCE_STRIPE" + } +} + +resource "snowflake_grant_privileges_to_account_role" "dlt_loader_role_t_role_grant_schema_source_stripe_stripe_why_usage" { + account_role_name = "DLT_LOADER_ROLE__T_ROLE" + privileges = ["USAGE"] + on_schema { + schema_name = "\"SOURCE_STRIPE.STRIPE_WHY\"" + } +} + +resource "snowflake_grant_privileges_to_account_role" "dlt_loader_role_t_role_future_table_source_stripe_select_insert_update_delete_truncate" { + account_role_name = "DLT_LOADER_ROLE__T_ROLE" + privileges = ["SELECT", "INSERT", "UPDATE", "DELETE", "TRUNCATE"] + on_schema_object { + object_type_plural = "TABLES" + in_database = "SOURCE_STRIPE" + } + all_privileges = false + with_grant_option = false +} + +resource "snowflake_grant_privileges_to_account_role" "dlt_loader_role_t_role_future_view_source_stripe_select" { + account_role_name = "DLT_LOADER_ROLE__T_ROLE" + privileges = ["SELECT"] + on_schema_object { + object_type_plural = "VIEWS" + in_database = "SOURCE_STRIPE" + } + all_privileges = false + with_grant_option = false +} + +resource "snowflake_grant_privileges_to_account_role" "dlt_stripe_tech_role_t_role_grant_database_source_stripe_usage_create_schema_monitor_modify" { + account_role_name = "DLT_STRIPE_TECH_ROLE__T_ROLE" + privileges = ["USAGE", "CREATE SCHEMA", "MONITOR", "MODIFY"] + on_account_object { + object_type = "DATABASE" + object_name = "SOURCE_STRIPE" + } +} + +resource "snowflake_grant_privileges_to_account_role" "dlt_stripe_tech_role_t_role_grant_database_proj_stripe_usage_create_schema_monitor_modify" { + account_role_name = "DLT_STRIPE_TECH_ROLE__T_ROLE" + privileges = ["USAGE", "CREATE SCHEMA", "MONITOR", "MODIFY"] + on_account_object { + object_type = "DATABASE" + object_name = "PROJ_STRIPE" + } +} + +resource "snowflake_grant_privileges_to_account_role" "dlt_stripe_tech_role_t_role_grant_schema_source_stripe_stripe_why_usage_modify_monitor_create_table_create_view_create_file_format_create_stage_create_sequence" { + account_role_name = "DLT_STRIPE_TECH_ROLE__T_ROLE" + privileges = ["USAGE", "MODIFY", "MONITOR", "CREATE TABLE", "CREATE VIEW", "CREATE FILE FORMAT", "CREATE STAGE", "CREATE SEQUENCE"] + on_schema { + schema_name = "\"SOURCE_STRIPE.STRIPE_WHY\"" + } +} + +resource "snowflake_grant_privileges_to_account_role" "dlt_stripe_tech_role_t_role_grant_warehouse_dlt_usage_monitor_operate" { + account_role_name = "DLT_STRIPE_TECH_ROLE__T_ROLE" + privileges = ["USAGE", "MONITOR", "OPERATE"] + on_account_object { + object_type = "WAREHOUSE" + object_name = "DLT" + } +} + +resource "snowflake_grant_privileges_to_account_role" "dlt_stripe_tech_role_t_role_future_file_format_source_stripe_usage" { + account_role_name = "DLT_STRIPE_TECH_ROLE__T_ROLE" + privileges = ["USAGE"] + on_schema_object { + object_type_plural = "FILE_FORMATS" + in_database = "SOURCE_STRIPE" + } + all_privileges = false + with_grant_option = false +} + +resource "snowflake_grant_privileges_to_account_role" "dlt_stripe_tech_role_t_role_future_file_format_proj_stripe_usage" { + account_role_name = "DLT_STRIPE_TECH_ROLE__T_ROLE" + privileges = ["USAGE"] + on_schema_object { + object_type_plural = "FILE_FORMATS" + in_database = "PROJ_STRIPE" + } + all_privileges = false + with_grant_option = false +} + +resource "snowflake_grant_privileges_to_account_role" "dlt_stripe_tech_role_t_role_future_sequence_source_stripe_usage" { + account_role_name = "DLT_STRIPE_TECH_ROLE__T_ROLE" + privileges = ["USAGE"] + on_schema_object { + object_type_plural = "SEQUENCES" + in_database = "SOURCE_STRIPE" + } + all_privileges = false + with_grant_option = false +} + +resource "snowflake_grant_privileges_to_account_role" "dlt_stripe_tech_role_t_role_future_sequence_proj_stripe_usage" { + account_role_name = "DLT_STRIPE_TECH_ROLE__T_ROLE" + privileges = ["USAGE"] + on_schema_object { + object_type_plural = "SEQUENCES" + in_database = "PROJ_STRIPE" + } + all_privileges = false + with_grant_option = false +} + +resource "snowflake_grant_privileges_to_account_role" "dlt_stripe_tech_role_t_role_future_stage_source_stripe_usage_read_write" { + account_role_name = "DLT_STRIPE_TECH_ROLE__T_ROLE" + privileges = ["USAGE", "READ", "WRITE"] + on_schema_object { + object_type_plural = "STAGES" + in_database = "SOURCE_STRIPE" + } + all_privileges = false + with_grant_option = false +} + +resource "snowflake_grant_privileges_to_account_role" "dlt_stripe_tech_role_t_role_future_stage_proj_stripe_usage_read_write" { + account_role_name = "DLT_STRIPE_TECH_ROLE__T_ROLE" + privileges = ["USAGE", "READ", "WRITE"] + on_schema_object { + object_type_plural = "STAGES" + in_database = "PROJ_STRIPE" + } + all_privileges = false + with_grant_option = false +} + +resource "snowflake_grant_privileges_to_account_role" "dlt_stripe_tech_role_t_role_future_table_source_stripe_select_insert_update_delete_truncate_references" { + account_role_name = "DLT_STRIPE_TECH_ROLE__T_ROLE" + privileges = ["SELECT", "INSERT", "UPDATE", "DELETE", "TRUNCATE", "REFERENCES"] + on_schema_object { + object_type_plural = "TABLES" + in_database = "SOURCE_STRIPE" + } + all_privileges = false + with_grant_option = false +} + +resource "snowflake_grant_privileges_to_account_role" "dlt_stripe_tech_role_t_role_future_table_proj_stripe_select_insert_update_delete_truncate_references" { + account_role_name = "DLT_STRIPE_TECH_ROLE__T_ROLE" + privileges = ["SELECT", "INSERT", "UPDATE", "DELETE", "TRUNCATE", "REFERENCES"] + on_schema_object { + object_type_plural = "TABLES" + in_database = "PROJ_STRIPE" + } + all_privileges = false + with_grant_option = false +} + +resource "snowflake_grant_privileges_to_account_role" "dlt_stripe_tech_role_t_role_future_view_source_stripe_select" { + account_role_name = "DLT_STRIPE_TECH_ROLE__T_ROLE" + privileges = ["SELECT"] + on_schema_object { + object_type_plural = "VIEWS" + in_database = "SOURCE_STRIPE" + } + all_privileges = false + with_grant_option = false +} + +resource "snowflake_grant_privileges_to_account_role" "dlt_stripe_tech_role_t_role_future_view_proj_stripe_select" { + account_role_name = "DLT_STRIPE_TECH_ROLE__T_ROLE" + privileges = ["SELECT"] + on_schema_object { + object_type_plural = "VIEWS" + in_database = "PROJ_STRIPE" + } + all_privileges = false + with_grant_option = false +} + +resource "snowflake_grant_privileges_to_account_role" "estuary_t_role_grant_warehouse_estuary_usage" { + account_role_name = "ESTUARY__T_ROLE" + privileges = ["USAGE"] + on_account_object { + object_type = "WAREHOUSE" + object_name = "ESTUARY" + } +} + +resource "snowflake_grant_privileges_to_account_role" "fabi_ai_t_role_grant_warehouse_fabi_ai_warehouse_usage" { + account_role_name = "FABI_AI__T_ROLE" + privileges = ["USAGE"] + on_account_object { + object_type = "WAREHOUSE" + object_name = "FABI_AI_WAREHOUSE" + } +} + +resource "snowflake_grant_privileges_to_account_role" "fivetran_t_role_grant_warehouse_fivetran_usage" { + account_role_name = "FIVETRAN__T_ROLE" + privileges = ["USAGE"] + on_account_object { + object_type = "WAREHOUSE" + object_name = "FIVETRAN" + } +} + +resource "snowflake_grant_privileges_to_account_role" "fivetran_role_t_role_grant_database_source_stripe_usage_create_schema" { + account_role_name = "FIVETRAN_ROLE__T_ROLE" + privileges = ["USAGE", "CREATE SCHEMA"] + on_account_object { + object_type = "DATABASE" + object_name = "SOURCE_STRIPE" + } +} + +resource "snowflake_grant_privileges_to_account_role" "fivetran_role_t_role_grant_schema_source_stripe_stripe_why_usage" { + account_role_name = "FIVETRAN_ROLE__T_ROLE" + privileges = ["USAGE"] + on_schema { + schema_name = "\"SOURCE_STRIPE.STRIPE_WHY\"" + } +} + +resource "snowflake_grant_privileges_to_account_role" "fivetran_role_t_role_grant_warehouse_fivetran_usage" { + account_role_name = "FIVETRAN_ROLE__T_ROLE" + privileges = ["USAGE"] + on_account_object { + object_type = "WAREHOUSE" + object_name = "FIVETRAN" + } +} + +resource "snowflake_grant_privileges_to_account_role" "fivetran_role_t_role_future_table_source_stripe_select_insert_update_delete_truncate" { + account_role_name = "FIVETRAN_ROLE__T_ROLE" + privileges = ["SELECT", "INSERT", "UPDATE", "DELETE", "TRUNCATE"] + on_schema_object { + object_type_plural = "TABLES" + in_database = "SOURCE_STRIPE" + } + all_privileges = false + with_grant_option = false +} + +resource "snowflake_grant_privileges_to_account_role" "lightdash_tech_role_t_role_grant_database_proj_stripe_usage" { + account_role_name = "LIGHTDASH_TECH_ROLE__T_ROLE" + privileges = ["USAGE"] + on_account_object { + object_type = "DATABASE" + object_name = "PROJ_STRIPE" + } +} + +resource "snowflake_grant_privileges_to_account_role" "lightdash_tech_role_t_role_grant_database_source_stripe_usage" { + account_role_name = "LIGHTDASH_TECH_ROLE__T_ROLE" + privileges = ["USAGE"] + on_account_object { + object_type = "DATABASE" + object_name = "SOURCE_STRIPE" + } +} + +resource "snowflake_grant_privileges_to_account_role" "lightdash_tech_role_t_role_grant_database_bi_tool_usage_create_schema" { + account_role_name = "LIGHTDASH_TECH_ROLE__T_ROLE" + privileges = ["USAGE", "CREATE SCHEMA"] + on_account_object { + object_type = "DATABASE" + object_name = "BI_TOOL" + } +} + +resource "snowflake_grant_privileges_to_account_role" "lightdash_tech_role_t_role_grant_schema_source_stripe_stripe_why_usage" { + account_role_name = "LIGHTDASH_TECH_ROLE__T_ROLE" + privileges = ["USAGE"] + on_schema { + schema_name = "\"SOURCE_STRIPE.STRIPE_WHY\"" + } +} + +resource "snowflake_grant_privileges_to_account_role" "lightdash_tech_role_t_role_grant_warehouse_bi_tool_usage" { + account_role_name = "LIGHTDASH_TECH_ROLE__T_ROLE" + privileges = ["USAGE"] + on_account_object { + object_type = "WAREHOUSE" + object_name = "BI_TOOL" + } +} + +resource "snowflake_grant_privileges_to_account_role" "lightdash_tech_role_t_role_future_table_proj_stripe_select" { + account_role_name = "LIGHTDASH_TECH_ROLE__T_ROLE" + privileges = ["SELECT"] + on_schema_object { + object_type_plural = "TABLES" + in_database = "PROJ_STRIPE" + } + all_privileges = false + with_grant_option = false +} + +resource "snowflake_grant_privileges_to_account_role" "lightdash_tech_role_t_role_future_table_source_stripe_select" { + account_role_name = "LIGHTDASH_TECH_ROLE__T_ROLE" + privileges = ["SELECT"] + on_schema_object { + object_type_plural = "TABLES" + in_database = "SOURCE_STRIPE" + } + all_privileges = false + with_grant_option = false +} + +resource "snowflake_grant_privileges_to_account_role" "lightdash_tech_role_t_role_future_table_bi_tool_select_insert_update_delete_truncate" { + account_role_name = "LIGHTDASH_TECH_ROLE__T_ROLE" + privileges = ["SELECT", "INSERT", "UPDATE", "DELETE", "TRUNCATE"] + on_schema_object { + object_type_plural = "TABLES" + in_database = "BI_TOOL" + } + all_privileges = false + with_grant_option = false +} + +resource "snowflake_grant_privileges_to_account_role" "lightdash_tech_role_t_role_future_view_proj_stripe_select" { + account_role_name = "LIGHTDASH_TECH_ROLE__T_ROLE" + privileges = ["SELECT"] + on_schema_object { + object_type_plural = "VIEWS" + in_database = "PROJ_STRIPE" + } + all_privileges = false + with_grant_option = false +} + +resource "snowflake_grant_privileges_to_account_role" "lightdash_tech_role_t_role_future_view_source_stripe_select" { + account_role_name = "LIGHTDASH_TECH_ROLE__T_ROLE" + privileges = ["SELECT"] + on_schema_object { + object_type_plural = "VIEWS" + in_database = "SOURCE_STRIPE" + } + all_privileges = false + with_grant_option = false +} + +resource "snowflake_grant_privileges_to_account_role" "lightdash_tech_role_t_role_future_view_bi_tool_select" { + account_role_name = "LIGHTDASH_TECH_ROLE__T_ROLE" + privileges = ["SELECT"] + on_schema_object { + object_type_plural = "VIEWS" + in_database = "BI_TOOL" + } + all_privileges = false + with_grant_option = false +} + +resource "snowflake_grant_privileges_to_account_role" "matillion_role_t_role_grant_database_source_stripe_usage_create_schema" { + account_role_name = "MATILLION_ROLE__T_ROLE" + privileges = ["USAGE", "CREATE SCHEMA"] + on_account_object { + object_type = "DATABASE" + object_name = "SOURCE_STRIPE" + } +} + +resource "snowflake_grant_privileges_to_account_role" "matillion_role_t_role_grant_database_proj_stripe_usage_create_schema" { + account_role_name = "MATILLION_ROLE__T_ROLE" + privileges = ["USAGE", "CREATE SCHEMA"] + on_account_object { + object_type = "DATABASE" + object_name = "PROJ_STRIPE" + } +} + +resource "snowflake_grant_privileges_to_account_role" "matillion_role_t_role_grant_schema_source_stripe_stripe_why_usage" { + account_role_name = "MATILLION_ROLE__T_ROLE" + privileges = ["USAGE"] + on_schema { + schema_name = "\"SOURCE_STRIPE.STRIPE_WHY\"" + } +} + +resource "snowflake_grant_privileges_to_account_role" "matillion_role_t_role_grant_warehouse_transforming_usage" { + account_role_name = "MATILLION_ROLE__T_ROLE" + privileges = ["USAGE"] + on_account_object { + object_type = "WAREHOUSE" + object_name = "TRANSFORMING" + } +} + +resource "snowflake_grant_privileges_to_account_role" "matillion_role_t_role_future_table_source_stripe_select_insert_update_delete_truncate" { + account_role_name = "MATILLION_ROLE__T_ROLE" + privileges = ["SELECT", "INSERT", "UPDATE", "DELETE", "TRUNCATE"] + on_schema_object { + object_type_plural = "TABLES" + in_database = "SOURCE_STRIPE" + } + all_privileges = false + with_grant_option = false +} + +resource "snowflake_grant_privileges_to_account_role" "matillion_role_t_role_future_table_proj_stripe_select_insert_update_delete_truncate" { + account_role_name = "MATILLION_ROLE__T_ROLE" + privileges = ["SELECT", "INSERT", "UPDATE", "DELETE", "TRUNCATE"] + on_schema_object { + object_type_plural = "TABLES" + in_database = "PROJ_STRIPE" + } + all_privileges = false + with_grant_option = false +} + +resource "snowflake_grant_privileges_to_account_role" "omni_infrastructure_automation_t_role_grant_database_analytics_tool_usage_create_schema_monitor" { + account_role_name = "OMNI_INFRASTRUCTURE_AUTOMATION__T_ROLE" + privileges = ["USAGE", "CREATE SCHEMA", "MONITOR"] + on_account_object { + object_type = "DATABASE" + object_name = "ANALYTICS_TOOL" + } +} + +resource "snowflake_grant_privileges_to_account_role" "omni_infrastructure_automation_t_role_grant_database_proj_stripe_usage_create_schema_monitor" { + account_role_name = "OMNI_INFRASTRUCTURE_AUTOMATION__T_ROLE" + privileges = ["USAGE", "CREATE SCHEMA", "MONITOR"] + on_account_object { + object_type = "DATABASE" + object_name = "PROJ_STRIPE" + } +} + +resource "snowflake_grant_privileges_to_account_role" "omni_infrastructure_automation_t_role_grant_warehouse_analytics_tool_usage_monitor_operate" { + account_role_name = "OMNI_INFRASTRUCTURE_AUTOMATION__T_ROLE" + privileges = ["USAGE", "MONITOR", "OPERATE"] + on_account_object { + object_type = "WAREHOUSE" + object_name = "ANALYTICS_TOOL" + } +} + +resource "snowflake_grant_privileges_to_account_role" "omni_infrastructure_automation_t_role_grant_warehouse_admin_usage_monitor_operate" { + account_role_name = "OMNI_INFRASTRUCTURE_AUTOMATION__T_ROLE" + privileges = ["USAGE", "MONITOR", "OPERATE"] + on_account_object { + object_type = "WAREHOUSE" + object_name = "ADMIN" + } +} + +resource "snowflake_grant_privileges_to_account_role" "omni_infrastructure_automation_t_role_future_table_analytics_tool_select_insert_update_delete_truncate" { + account_role_name = "OMNI_INFRASTRUCTURE_AUTOMATION__T_ROLE" + privileges = ["SELECT", "INSERT", "UPDATE", "DELETE", "TRUNCATE"] + on_schema_object { + object_type_plural = "TABLES" + in_database = "ANALYTICS_TOOL" + } + all_privileges = false + with_grant_option = false +} + +resource "snowflake_grant_privileges_to_account_role" "omni_infrastructure_automation_t_role_future_view_analytics_tool_select" { + account_role_name = "OMNI_INFRASTRUCTURE_AUTOMATION__T_ROLE" + privileges = ["SELECT"] + on_schema_object { + object_type_plural = "VIEWS" + in_database = "ANALYTICS_TOOL" + } + all_privileges = false + with_grant_option = false +} + +resource "snowflake_grant_privileges_to_account_role" "omni_infrastructure_automation_t_role_future_view_proj_stripe_select" { + account_role_name = "OMNI_INFRASTRUCTURE_AUTOMATION__T_ROLE" + privileges = ["SELECT"] + on_schema_object { + object_type_plural = "VIEWS" + in_database = "PROJ_STRIPE" + } + all_privileges = false + with_grant_option = false +} + +resource "snowflake_grant_privileges_to_account_role" "omni_tech_role_t_role_grant_database_proj_stripe_usage" { + account_role_name = "OMNI_TECH_ROLE__T_ROLE" + privileges = ["USAGE"] + on_account_object { + object_type = "DATABASE" + object_name = "PROJ_STRIPE" + } +} + +resource "snowflake_grant_privileges_to_account_role" "omni_tech_role_t_role_grant_database_analytics_tool_usage_create_schema" { + account_role_name = "OMNI_TECH_ROLE__T_ROLE" + privileges = ["USAGE", "CREATE SCHEMA"] + on_account_object { + object_type = "DATABASE" + object_name = "ANALYTICS_TOOL" + } +} + +resource "snowflake_grant_privileges_to_account_role" "omni_tech_role_t_role_grant_warehouse_analytics_tool_usage" { + account_role_name = "OMNI_TECH_ROLE__T_ROLE" + privileges = ["USAGE"] + on_account_object { + object_type = "WAREHOUSE" + object_name = "ANALYTICS_TOOL" + } +} + +resource "snowflake_grant_privileges_to_account_role" "omni_tech_role_t_role_future_table_proj_stripe_select" { + account_role_name = "OMNI_TECH_ROLE__T_ROLE" + privileges = ["SELECT"] + on_schema_object { + object_type_plural = "TABLES" + in_database = "PROJ_STRIPE" + } + all_privileges = false + with_grant_option = false +} + +resource "snowflake_grant_privileges_to_account_role" "omni_tech_role_t_role_future_table_analytics_tool_select_insert_update_delete_truncate" { + account_role_name = "OMNI_TECH_ROLE__T_ROLE" + privileges = ["SELECT", "INSERT", "UPDATE", "DELETE", "TRUNCATE"] + on_schema_object { + object_type_plural = "TABLES" + in_database = "ANALYTICS_TOOL" + } + all_privileges = false + with_grant_option = false +} + +resource "snowflake_grant_privileges_to_account_role" "omni_tech_role_t_role_future_view_analytics_tool_select" { + account_role_name = "OMNI_TECH_ROLE__T_ROLE" + privileges = ["SELECT"] + on_schema_object { + object_type_plural = "VIEWS" + in_database = "ANALYTICS_TOOL" + } + all_privileges = false + with_grant_option = false +} + +resource "snowflake_grant_privileges_to_account_role" "recce_tech_role_t_role_grant_database_proj_stripe_usage" { + account_role_name = "RECCE_TECH_ROLE__T_ROLE" + privileges = ["USAGE"] + on_account_object { + object_type = "DATABASE" + object_name = "PROJ_STRIPE" + } +} + +resource "snowflake_grant_privileges_to_account_role" "recce_tech_role_t_role_grant_warehouse_recce_usage" { + account_role_name = "RECCE_TECH_ROLE__T_ROLE" + privileges = ["USAGE"] + on_account_object { + object_type = "WAREHOUSE" + object_name = "RECCE" + } +} + +resource "snowflake_grant_privileges_to_account_role" "recce_tech_role_t_role_future_table_proj_stripe_select" { + account_role_name = "RECCE_TECH_ROLE__T_ROLE" + privileges = ["SELECT"] + on_schema_object { + object_type_plural = "TABLES" + in_database = "PROJ_STRIPE" + } + all_privileges = false + with_grant_option = false +} + +resource "snowflake_grant_privileges_to_account_role" "recce_tech_role_t_role_future_view_proj_stripe_select" { + account_role_name = "RECCE_TECH_ROLE__T_ROLE" + privileges = ["SELECT"] + on_schema_object { + object_type_plural = "VIEWS" + in_database = "PROJ_STRIPE" + } + all_privileges = false + with_grant_option = false +} + +resource "snowflake_grant_privileges_to_account_role" "sigma_role_t_role_grant_database_proj_stripe_usage" { + account_role_name = "SIGMA_ROLE__T_ROLE" + privileges = ["USAGE"] + on_account_object { + object_type = "DATABASE" + object_name = "PROJ_STRIPE" + } +} + +resource "snowflake_grant_privileges_to_account_role" "sigma_role_t_role_grant_database_analytics_tool_usage" { + account_role_name = "SIGMA_ROLE__T_ROLE" + privileges = ["USAGE"] + on_account_object { + object_type = "DATABASE" + object_name = "ANALYTICS_TOOL" + } +} + +resource "snowflake_grant_privileges_to_account_role" "sigma_role_t_role_grant_warehouse_analytics_tool_usage" { + account_role_name = "SIGMA_ROLE__T_ROLE" + privileges = ["USAGE"] + on_account_object { + object_type = "WAREHOUSE" + object_name = "ANALYTICS_TOOL" + } +} + +resource "snowflake_grant_privileges_to_account_role" "sigma_role_t_role_future_table_proj_stripe_select" { + account_role_name = "SIGMA_ROLE__T_ROLE" + privileges = ["SELECT"] + on_schema_object { + object_type_plural = "TABLES" + in_database = "PROJ_STRIPE" + } + all_privileges = false + with_grant_option = false +} + +resource "snowflake_grant_privileges_to_account_role" "sigma_role_t_role_future_table_analytics_tool_select" { + account_role_name = "SIGMA_ROLE__T_ROLE" + privileges = ["SELECT"] + on_schema_object { + object_type_plural = "TABLES" + in_database = "ANALYTICS_TOOL" + } + all_privileges = false + with_grant_option = false +} + +resource "snowflake_grant_privileges_to_account_role" "sigma_role_t_role_future_view_proj_stripe_select" { + account_role_name = "SIGMA_ROLE__T_ROLE" + privileges = ["SELECT"] + on_schema_object { + object_type_plural = "VIEWS" + in_database = "PROJ_STRIPE" + } + all_privileges = false + with_grant_option = false +} + +resource "snowflake_grant_privileges_to_account_role" "sigma_role_t_role_future_view_analytics_tool_select" { + account_role_name = "SIGMA_ROLE__T_ROLE" + privileges = ["SELECT"] + on_schema_object { + object_type_plural = "VIEWS" + in_database = "ANALYTICS_TOOL" + } + all_privileges = false + with_grant_option = false +} + +resource "snowflake_grant_privileges_to_account_role" "snowtower_users_t_role_grant_warehouse_main_warehouse_usage" { + account_role_name = "SNOWTOWER_USERS__T_ROLE" + privileges = ["USAGE"] + on_account_object { + object_type = "WAREHOUSE" + object_name = "MAIN_WAREHOUSE" + } +} + +resource "snowflake_grant_privileges_to_account_role" "streamlit_role_t_role_grant_database_proj_stripe_usage" { + account_role_name = "STREAMLIT_ROLE__T_ROLE" + privileges = ["USAGE"] + on_account_object { + object_type = "DATABASE" + object_name = "PROJ_STRIPE" + } +} + +resource "snowflake_grant_privileges_to_account_role" "streamlit_role_t_role_grant_warehouse_streamlit_viewer_wh_usage" { + account_role_name = "STREAMLIT_ROLE__T_ROLE" + privileges = ["USAGE"] + on_account_object { + object_type = "WAREHOUSE" + object_name = "STREAMLIT_VIEWER_WH" + } +} + +resource "snowflake_grant_privileges_to_account_role" "streamlit_role_t_role_future_table_proj_stripe_select" { + account_role_name = "STREAMLIT_ROLE__T_ROLE" + privileges = ["SELECT"] + on_schema_object { + object_type_plural = "TABLES" + in_database = "PROJ_STRIPE" + } + all_privileges = false + with_grant_option = false +} + +resource "snowflake_grant_privileges_to_account_role" "streamlit_role_t_role_future_view_proj_stripe_select" { + account_role_name = "STREAMLIT_ROLE__T_ROLE" + privileges = ["SELECT"] + on_schema_object { + object_type_plural = "VIEWS" + in_database = "PROJ_STRIPE" + } + all_privileges = false + with_grant_option = false +} + +resource "snowflake_grant_privileges_to_account_role" "streamlit_towerapp_role_t_role_grant_database_proj_stripe_usage" { + account_role_name = "STREAMLIT_TOWERAPP_ROLE__T_ROLE" + privileges = ["USAGE"] + on_account_object { + object_type = "DATABASE" + object_name = "PROJ_STRIPE" + } +} + +resource "snowflake_grant_privileges_to_account_role" "streamlit_towerapp_role_t_role_grant_database_source_stripe_usage" { + account_role_name = "STREAMLIT_TOWERAPP_ROLE__T_ROLE" + privileges = ["USAGE"] + on_account_object { + object_type = "DATABASE" + object_name = "SOURCE_STRIPE" + } +} + +resource "snowflake_grant_privileges_to_account_role" "streamlit_towerapp_role_t_role_grant_schema_source_stripe_stripe_why_usage" { + account_role_name = "STREAMLIT_TOWERAPP_ROLE__T_ROLE" + privileges = ["USAGE"] + on_schema { + schema_name = "\"SOURCE_STRIPE.STRIPE_WHY\"" + } +} + +resource "snowflake_grant_privileges_to_account_role" "streamlit_towerapp_role_t_role_grant_warehouse_streamlit_viewer_wh_usage" { + account_role_name = "STREAMLIT_TOWERAPP_ROLE__T_ROLE" + privileges = ["USAGE"] + on_account_object { + object_type = "WAREHOUSE" + object_name = "STREAMLIT_VIEWER_WH" + } +} + +resource "snowflake_grant_privileges_to_account_role" "streamlit_towerapp_role_t_role_grant_warehouse_main_warehouse_usage" { + account_role_name = "STREAMLIT_TOWERAPP_ROLE__T_ROLE" + privileges = ["USAGE"] + on_account_object { + object_type = "WAREHOUSE" + object_name = "MAIN_WAREHOUSE" + } +} + +resource "snowflake_grant_privileges_to_account_role" "streamlit_towerapp_role_t_role_future_table_proj_stripe_select" { + account_role_name = "STREAMLIT_TOWERAPP_ROLE__T_ROLE" + privileges = ["SELECT"] + on_schema_object { + object_type_plural = "TABLES" + in_database = "PROJ_STRIPE" + } + all_privileges = false + with_grant_option = false +} + +resource "snowflake_grant_privileges_to_account_role" "streamlit_towerapp_role_t_role_future_table_source_stripe_select" { + account_role_name = "STREAMLIT_TOWERAPP_ROLE__T_ROLE" + privileges = ["SELECT"] + on_schema_object { + object_type_plural = "TABLES" + in_database = "SOURCE_STRIPE" + } + all_privileges = false + with_grant_option = false +} + +resource "snowflake_grant_privileges_to_account_role" "streamlit_towerapp_role_t_role_future_view_proj_stripe_select" { + account_role_name = "STREAMLIT_TOWERAPP_ROLE__T_ROLE" + privileges = ["SELECT"] + on_schema_object { + object_type_plural = "VIEWS" + in_database = "PROJ_STRIPE" + } + all_privileges = false + with_grant_option = false +} + +resource "snowflake_grant_privileges_to_account_role" "streamlit_towerapp_role_t_role_future_view_source_stripe_select" { + account_role_name = "STREAMLIT_TOWERAPP_ROLE__T_ROLE" + privileges = ["SELECT"] + on_schema_object { + object_type_plural = "VIEWS" + in_database = "SOURCE_STRIPE" + } + all_privileges = false + with_grant_option = false +} + +resource "snowflake_grant_privileges_to_account_role" "streamlit_viewer_t_role_grant_warehouse_streamlit_viewer_wh_usage" { + account_role_name = "STREAMLIT_VIEWER__T_ROLE" + privileges = ["USAGE"] + on_account_object { + object_type = "WAREHOUSE" + object_name = "STREAMLIT_VIEWER_WH" + } +} + +resource "snowflake_grant_privileges_to_account_role" "stripe_t_role_grant_database_source_stripe_usage_create_schema" { + account_role_name = "STRIPE__T_ROLE" + privileges = ["USAGE", "CREATE SCHEMA"] + on_account_object { + object_type = "DATABASE" + object_name = "SOURCE_STRIPE" + } +} + +resource "snowflake_grant_privileges_to_account_role" "stripe_t_role_grant_database_proj_stripe_usage_create_schema" { + account_role_name = "STRIPE__T_ROLE" + privileges = ["USAGE", "CREATE SCHEMA"] + on_account_object { + object_type = "DATABASE" + object_name = "PROJ_STRIPE" + } +} + +resource "snowflake_grant_privileges_to_account_role" "stripe_t_role_grant_schema_source_stripe_stripe_why_usage" { + account_role_name = "STRIPE__T_ROLE" + privileges = ["USAGE"] + on_schema { + schema_name = "\"SOURCE_STRIPE.STRIPE_WHY\"" + } +} + +resource "snowflake_grant_privileges_to_account_role" "stripe_t_role_future_table_source_stripe_select" { + account_role_name = "STRIPE__T_ROLE" + privileges = ["SELECT"] + on_schema_object { + object_type_plural = "TABLES" + in_database = "SOURCE_STRIPE" + } + all_privileges = false + with_grant_option = false +} + +resource "snowflake_grant_privileges_to_account_role" "stripe_t_role_future_table_proj_stripe_select" { + account_role_name = "STRIPE__T_ROLE" + privileges = ["SELECT"] + on_schema_object { + object_type_plural = "TABLES" + in_database = "PROJ_STRIPE" + } + all_privileges = false + with_grant_option = false +} + +resource "snowflake_grant_privileges_to_account_role" "stripe_t_role_future_view_source_stripe_select" { + account_role_name = "STRIPE__T_ROLE" + privileges = ["SELECT"] + on_schema_object { + object_type_plural = "VIEWS" + in_database = "SOURCE_STRIPE" + } + all_privileges = false + with_grant_option = false +} + +resource "snowflake_grant_privileges_to_account_role" "stripe_t_role_future_view_proj_stripe_select" { + account_role_name = "STRIPE__T_ROLE" + privileges = ["SELECT"] + on_schema_object { + object_type_plural = "VIEWS" + in_database = "PROJ_STRIPE" + } + all_privileges = false + with_grant_option = false +} + +resource "snowflake_grant_privileges_to_account_role" "tobiko_cloud_t_role_grant_warehouse_admin_usage" { + account_role_name = "TOBIKO_CLOUD__T_ROLE" + privileges = ["USAGE"] + on_account_object { + object_type = "WAREHOUSE" + object_name = "ADMIN" + } +} diff --git a/terraform-snowflake-boilerplate/main.tf b/terraform-snowflake-boilerplate/main.tf new file mode 100644 index 0000000..149d2e2 --- /dev/null +++ b/terraform-snowflake-boilerplate/main.tf @@ -0,0 +1,11 @@ +# Generated by SnowTower - DO NOT EDIT MANUALLY +# Source: snowddl/ YAML configurations + +terraform { + required_providers { + snowflake = { + source = "Snowflake-Labs/snowflake" + version = "~> 1.0" + } + } +} diff --git a/terraform-snowflake-boilerplate/policies.tf b/terraform-snowflake-boilerplate/policies.tf new file mode 100644 index 0000000..e8e1167 --- /dev/null +++ b/terraform-snowflake-boilerplate/policies.tf @@ -0,0 +1,194 @@ +# Generated by SnowTower - DO NOT EDIT MANUALLY +# Source: snowddl/ YAML configurations + +# Policy resources (network, authentication, password, session) + +import { + to = snowflake_network_policy.analytics_network_policy + id = "ANALYTICS_NETWORK_POLICY" +} + +resource "snowflake_network_policy" "analytics_network_policy" { + name = "ANALYTICS_NETWORK_POLICY" + allowed_ip_list = ["3.211.115.21/32", "34.215.77.3/32", "44.199.154.198/32", "52.205.119.255/32", "52.33.167.28/32", "54.172.168.164/32", "54.213.140.155/32", "54.213.219.131/32"] + comment = "Restrict Omni BI platform service account to approved Omni IP addresses" +} + +import { + to = snowflake_network_policy.bi_tool_network_policy + id = "BI_TOOL_NETWORK_POLICY" +} + +resource "snowflake_network_policy" "bi_tool_network_policy" { + name = "BI_TOOL_NETWORK_POLICY" + allowed_ip_list = ["198.51.100.1/32", "198.51.100.2/32", "192.0.2.10/32"] + comment = "Restrict LightDash BI platform service account to approved LightDash IP addresses (US and EU regions)" +} + +import { + to = snowflake_network_policy.company_network_policy + id = "COMPANY_NETWORK_POLICY" +} + +resource "snowflake_network_policy" "company_network_policy" { + name = "COMPANY_NETWORK_POLICY" + allowed_ip_list = ["192.0.2.10/32"] + comment = "Restrict human user access to approved IP address only" +} + +import { + to = snowflake_authentication_policy.mfa_optional_policy + id = "MFA_OPTIONAL_POLICY" +} + +resource "snowflake_authentication_policy" "mfa_optional_policy" { + name = "MFA_OPTIONAL_POLICY" + comment = "Optional MFA policy for transition period before mandatory enforcement" + client_types = ["ALL"] + mfa_authentication_methods = ["PASSWORD", "SAML"] + mfa_enrollment = "OPTIONAL" +} + +import { + to = snowflake_authentication_policy.mfa_required_policy + id = "MFA_REQUIRED_POLICY" +} + +resource "snowflake_authentication_policy" "mfa_required_policy" { + name = "MFA_REQUIRED_POLICY" + comment = "Mandatory MFA policy for human users (TYPE=PERSON) to comply with Snowflake MFA rollout timeline" + client_types = ["SNOWFLAKE_UI", "SNOWFLAKE_CLI", "DRIVERS", "SNOWSQL"] + mfa_authentication_methods = ["PASSWORD"] + mfa_enrollment = "REQUIRED" +} + +import { + to = snowflake_authentication_policy.service_account_policy + id = "SERVICE_ACCOUNT_POLICY" +} + +resource "snowflake_authentication_policy" "service_account_policy" { + name = "SERVICE_ACCOUNT_POLICY" + comment = "Restricted client access policy for service accounts using key-pair authentication" + client_types = ["DRIVERS", "SNOWFLAKE_CLI", "SNOWSQL"] +} + +import { + to = snowflake_password_policy.admin_password_policy + id = "ADMIN_PASSWORD_POLICY" +} + +resource "snowflake_password_policy" "admin_password_policy" { + name = "ADMIN_PASSWORD_POLICY" + comment = "Enhanced password security for admin users" + password_history = 15 + password_lockout_time_mins = 60 + password_max_age_days = 60 + password_max_length = 256 + password_max_retries = 3 + password_min_length = 16 + password_min_lower_case_chars = 2 + password_min_numeric_chars = 2 + password_min_special_chars = 2 + password_min_upper_case_chars = 2 +} + +import { + to = snowflake_password_policy.service_password_policy + id = "SERVICE_PASSWORD_POLICY" +} + +resource "snowflake_password_policy" "service_password_policy" { + name = "SERVICE_PASSWORD_POLICY" + comment = "Temporary password policy for service accounts (migrate to RSA keys)" + password_history = 20 + password_lockout_time_mins = 120 + password_max_age_days = 180 + password_max_length = 256 + password_max_retries = 3 + password_min_length = 20 + password_min_lower_case_chars = 2 + password_min_numeric_chars = 3 + password_min_special_chars = 2 + password_min_upper_case_chars = 2 +} + +import { + to = snowflake_password_policy.standard_password_policy + id = "STANDARD_PASSWORD_POLICY" +} + +resource "snowflake_password_policy" "standard_password_policy" { + name = "STANDARD_PASSWORD_POLICY" + comment = "Standard password security for regular users" + password_history = 12 + password_lockout_time_mins = 30 + password_max_age_days = 90 + password_max_length = 256 + password_max_retries = 5 + password_min_length = 14 + password_min_lower_case_chars = 1 + password_min_numeric_chars = 2 + password_min_special_chars = 1 + password_min_upper_case_chars = 1 +} + +import { + to = snowflake_session_policy.admin_session_policy + id = "ADMIN_SESSION_POLICY" +} + +resource "snowflake_session_policy" "admin_session_policy" { + name = "ADMIN_SESSION_POLICY" + comment = "Strict session policy for admin users" + session_idle_timeout_mins = 15 + session_ui_idle_timeout_mins = 15 +} + +import { + to = snowflake_session_policy.analyst_session_policy + id = "ANALYST_SESSION_POLICY" +} + +resource "snowflake_session_policy" "analyst_session_policy" { + name = "ANALYST_SESSION_POLICY" + comment = "Extended session policy for data analysts running long queries" + session_idle_timeout_mins = 180 + session_ui_idle_timeout_mins = 240 +} + +import { + to = snowflake_session_policy.dev_session_policy + id = "DEV_SESSION_POLICY" +} + +resource "snowflake_session_policy" "dev_session_policy" { + name = "DEV_SESSION_POLICY" + comment = "Development environment session policy" + session_idle_timeout_mins = 120 + session_ui_idle_timeout_mins = 180 +} + +import { + to = snowflake_session_policy.service_session_policy + id = "SERVICE_SESSION_POLICY" +} + +resource "snowflake_session_policy" "service_session_policy" { + name = "SERVICE_SESSION_POLICY" + comment = "Session policy for service accounts and automation" + session_idle_timeout_mins = 240 + session_ui_idle_timeout_mins = 60 +} + +import { + to = snowflake_session_policy.user_session_policy + id = "USER_SESSION_POLICY" +} + +resource "snowflake_session_policy" "user_session_policy" { + name = "USER_SESSION_POLICY" + comment = "Standard session policy for regular users" + session_idle_timeout_mins = 60 + session_ui_idle_timeout_mins = 120 +} diff --git a/terraform-snowflake-boilerplate/resource_monitors.tf b/terraform-snowflake-boilerplate/resource_monitors.tf new file mode 100644 index 0000000..a213cf1 --- /dev/null +++ b/terraform-snowflake-boilerplate/resource_monitors.tf @@ -0,0 +1,107 @@ +# Generated by SnowTower - DO NOT EDIT MANUALLY +# Source: snowddl/ YAML configurations + +# Resource monitor resources + +import { + to = snowflake_resource_monitor.analyst_warehouse_monitor + id = "ANALYST_WAREHOUSE_MONITOR" +} + +resource "snowflake_resource_monitor" "analyst_warehouse_monitor" { + name = "ANALYST_WAREHOUSE_MONITOR" + credit_quota = 100 + frequency = "MONTHLY" + notify_triggers = [75, 90] + suspend_triggers = [100] +} + +import { + to = snowflake_resource_monitor.dev_environment_monitor + id = "DEV_ENVIRONMENT_MONITOR" +} + +resource "snowflake_resource_monitor" "dev_environment_monitor" { + name = "DEV_ENVIRONMENT_MONITOR" + credit_quota = 500 + frequency = "MONTHLY" + notify_triggers = [60, 80] + suspend_triggers = [95] +} + +import { + to = snowflake_resource_monitor.dlt_pipeline_monitor + id = "DLT_PIPELINE_MONITOR" +} + +resource "snowflake_resource_monitor" "dlt_pipeline_monitor" { + name = "DLT_PIPELINE_MONITOR" + credit_quota = 300 + frequency = "MONTHLY" + notify_triggers = [80, 95] + suspend_triggers = [100] +} + +import { + to = snowflake_resource_monitor.emergency_cost_control + id = "EMERGENCY_COST_CONTROL" +} + +resource "snowflake_resource_monitor" "emergency_cost_control" { + name = "EMERGENCY_COST_CONTROL" + credit_quota = 1500 + frequency = "MONTHLY" + notify_triggers = [95] + suspend_triggers = [100] +} + +import { + to = snowflake_resource_monitor.pipeline_monitor + id = "PIPELINE_MONITOR" +} + +resource "snowflake_resource_monitor" "pipeline_monitor" { + name = "PIPELINE_MONITOR" + credit_quota = 500 + frequency = "MONTHLY" + notify_triggers = [70, 85] + suspend_triggers = [98] +} + +import { + to = snowflake_resource_monitor.snowtower_account_monitor + id = "SNOWTOWER_ACCOUNT_MONITOR" +} + +resource "snowflake_resource_monitor" "snowtower_account_monitor" { + name = "SNOWTOWER_ACCOUNT_MONITOR" + credit_quota = 1000 + frequency = "MONTHLY" + notify_triggers = [50, 75, 90] + suspend_triggers = [100] +} + +import { + to = snowflake_resource_monitor.snowtower_monitor + id = "SNOWTOWER_MONITOR" +} + +resource "snowflake_resource_monitor" "snowtower_monitor" { + name = "SNOWTOWER_MONITOR" + credit_quota = 50 + frequency = "MONTHLY" + notify_triggers = [75, 90] + suspend_triggers = [100] +} + +import { + to = snowflake_resource_monitor.weekly_spending_monitor + id = "WEEKLY_SPENDING_MONITOR" +} + +resource "snowflake_resource_monitor" "weekly_spending_monitor" { + name = "WEEKLY_SPENDING_MONITOR" + credit_quota = 150 + frequency = "WEEKLY" + notify_triggers = [80] +} diff --git a/terraform-snowflake-boilerplate/roles.tf b/terraform-snowflake-boilerplate/roles.tf new file mode 100644 index 0000000..b8efbc9 --- /dev/null +++ b/terraform-snowflake-boilerplate/roles.tf @@ -0,0 +1,374 @@ +# Generated by SnowTower - DO NOT EDIT MANUALLY +# Source: snowddl/ YAML configurations + +# Role resources (business + tech) + +import { + to = snowflake_account_role.admin_role_b_role + id = "ADMIN_ROLE__B_ROLE" +} + +resource "snowflake_account_role" "admin_role_b_role" { + name = "ADMIN_ROLE__B_ROLE" + comment = "Administrative business role with full access" +} + +import { + to = snowflake_account_role.ai_ml_role_b_role + id = "AI_ML_ROLE__B_ROLE" +} + +resource "snowflake_account_role" "ai_ml_role_b_role" { + name = "AI_ML_ROLE__B_ROLE" + comment = "Business role for AI and ML workloads" +} + +import { + to = snowflake_account_role.bi_developer_role_b_role + id = "BI_DEVELOPER_ROLE__B_ROLE" +} + +resource "snowflake_account_role" "bi_developer_role_b_role" { + name = "BI_DEVELOPER_ROLE__B_ROLE" + comment = "Business role for BI developers who create models and dashboards in ANALYTICS_TOOL using dbt transformations" +} + +import { + to = snowflake_account_role.company_users_b_role + id = "COMPANY_USERS__B_ROLE" +} + +resource "snowflake_account_role" "company_users_b_role" { + name = "COMPANY_USERS__B_ROLE" + comment = "Business role for SnowTower team members doing data work" +} + +import { + to = snowflake_account_role.data_integration_role_b_role + id = "DATA_INTEGRATION_ROLE__B_ROLE" +} + +resource "snowflake_account_role" "data_integration_role_b_role" { + name = "DATA_INTEGRATION_ROLE__B_ROLE" + comment = "Business role for data integration services" +} + +import { + to = snowflake_account_role.dbt_analytics_role_b_role + id = "DBT_ANALYTICS_ROLE__B_ROLE" +} + +resource "snowflake_account_role" "dbt_analytics_role_b_role" { + name = "DBT_ANALYTICS_ROLE__B_ROLE" + comment = "Business role for dbt and analytics work - includes schema owner access for PROJ_STRIPE to modify existing tables/views" +} + +import { + to = snowflake_account_role.dlt_stripe_role_b_role + id = "DLT_STRIPE_ROLE__B_ROLE" +} + +resource "snowflake_account_role" "dlt_stripe_role_b_role" { + name = "DLT_STRIPE_ROLE__B_ROLE" + comment = "Dedicated business role for DLT Stripe data pipeline - includes schema owner access for table stage operations" +} + +import { + to = snowflake_account_role.infrastructure_automation_role_b_role + id = "INFRASTRUCTURE_AUTOMATION_ROLE__B_ROLE" +} + +resource "snowflake_account_role" "infrastructure_automation_role_b_role" { + name = "INFRASTRUCTURE_AUTOMATION_ROLE__B_ROLE" + comment = "Business role for infrastructure automation and deployment services" +} + +import { + to = snowflake_account_role.lightdash_business_role_b_role + id = "LIGHTDASH_BUSINESS_ROLE__B_ROLE" +} + +resource "snowflake_account_role" "lightdash_business_role_b_role" { + name = "LIGHTDASH_BUSINESS_ROLE__B_ROLE" + comment = "Business role for LightDash BI platform - analytics and dashboard access with read/write to BI_TOOL database" +} + +import { + to = snowflake_account_role.omni_business_role_b_role + id = "OMNI_BUSINESS_ROLE__B_ROLE" +} + +resource "snowflake_account_role" "omni_business_role_b_role" { + name = "OMNI_BUSINESS_ROLE__B_ROLE" + comment = "Business role for Omni BI platform - analytics and reporting access" +} + +import { + to = snowflake_account_role.recce_business_role_b_role + id = "RECCE_BUSINESS_ROLE__B_ROLE" +} + +resource "snowflake_account_role" "recce_business_role_b_role" { + name = "RECCE_BUSINESS_ROLE__B_ROLE" + comment = "Business role for Recce dbt validation and data quality testing" +} + +import { + to = snowflake_account_role.service_accounts_role_b_role + id = "SERVICE_ACCOUNTS_ROLE__B_ROLE" +} + +resource "snowflake_account_role" "service_accounts_role_b_role" { + name = "SERVICE_ACCOUNTS_ROLE__B_ROLE" + comment = "Business role for service accounts and integrations" +} + +import { + to = snowflake_account_role.streamlit_apps_role_b_role + id = "STREAMLIT_APPS_ROLE__B_ROLE" +} + +resource "snowflake_account_role" "streamlit_apps_role_b_role" { + name = "STREAMLIT_APPS_ROLE__B_ROLE" + comment = "Business role for Streamlit applications and services" +} + +import { + to = snowflake_account_role.training_role_b_role + id = "TRAINING_ROLE__B_ROLE" +} + +resource "snowflake_account_role" "training_role_b_role" { + name = "TRAINING_ROLE__B_ROLE" + comment = "Business role for training and development users" +} + +import { + to = snowflake_account_role.bi_writer_tech_role_t_role + id = "BI_WRITER_TECH_ROLE__T_ROLE" +} + +resource "snowflake_account_role" "bi_writer_tech_role_t_role" { + name = "BI_WRITER_TECH_ROLE__T_ROLE" + comment = "Technical role for BI developers who need to read from transformed data and write to ANALYTICS_TOOL" +} + +import { + to = snowflake_account_role.company_users_t_role + id = "COMPANY_USERS__T_ROLE" +} + +resource "snowflake_account_role" "company_users_t_role" { + name = "COMPANY_USERS__T_ROLE" + comment = "SnowTower team members role with development database access" +} + +import { + to = snowflake_account_role.copilot_access_role_t_role + id = "COPILOT_ACCESS_ROLE__T_ROLE" +} + +resource "snowflake_account_role" "copilot_access_role_t_role" { + name = "COPILOT_ACCESS_ROLE__T_ROLE" + comment = "Role for Copilot access and AI integrations" +} + +import { + to = snowflake_account_role.dbt_stripe_role_t_role + id = "DBT_STRIPE_ROLE__T_ROLE" +} + +resource "snowflake_account_role" "dbt_stripe_role_t_role" { + name = "DBT_STRIPE_ROLE__T_ROLE" + comment = "dbt service role for Stripe transformations - Read-only access to SOURCE_STRIPE (DLT-owned source data), full read/write access to PROJ_STRIPE and ANALYTICS_TOOL for transformed models and analytics" +} + +import { + to = snowflake_account_role.dbt_training_t_role + id = "DBT_TRAINING__T_ROLE" +} + +resource "snowflake_account_role" "dbt_training_t_role" { + name = "DBT_TRAINING__T_ROLE" + comment = "Training role for dbt users and courses" +} + +import { + to = snowflake_account_role.dlt_loader_role_t_role + id = "DLT_LOADER_ROLE__T_ROLE" +} + +resource "snowflake_account_role" "dlt_loader_role_t_role" { + name = "DLT_LOADER_ROLE__T_ROLE" + comment = "Data loading role for DLT operations with SOURCE_STRIPE permissions" +} + +import { + to = snowflake_account_role.dlt_stripe_tech_role_t_role + id = "DLT_STRIPE_TECH_ROLE__T_ROLE" +} + +resource "snowflake_account_role" "dlt_stripe_tech_role_t_role" { + name = "DLT_STRIPE_TECH_ROLE__T_ROLE" + comment = "Dedicated technical role for DLT Stripe pipeline - comprehensive permissions for data loading operations including table stage access via schema owner role." +} + +import { + to = snowflake_account_role.estuary_t_role + id = "ESTUARY__T_ROLE" +} + +resource "snowflake_account_role" "estuary_t_role" { + name = "ESTUARY__T_ROLE" + comment = "Estuary data integration service role" +} + +import { + to = snowflake_account_role.fabi_ai_t_role + id = "FABI_AI__T_ROLE" +} + +resource "snowflake_account_role" "fabi_ai_t_role" { + name = "FABI_AI__T_ROLE" + comment = "Fabi AI service role for AI/ML operations" +} + +import { + to = snowflake_account_role.fivetran_t_role + id = "FIVETRAN__T_ROLE" +} + +resource "snowflake_account_role" "fivetran_t_role" { + name = "FIVETRAN__T_ROLE" + comment = "Fivetran data integration service role" +} + +import { + to = snowflake_account_role.fivetran_role_t_role + id = "FIVETRAN_ROLE__T_ROLE" +} + +resource "snowflake_account_role" "fivetran_role_t_role" { + name = "FIVETRAN_ROLE__T_ROLE" + comment = "Alternative Fivetran role for data integration operations" +} + +import { + to = snowflake_account_role.lightdash_tech_role_t_role + id = "LIGHTDASH_TECH_ROLE__T_ROLE" +} + +resource "snowflake_account_role" "lightdash_tech_role_t_role" { + name = "LIGHTDASH_TECH_ROLE__T_ROLE" + comment = "LightDash BI platform technical role - Read access to source data, full read/write access to BI_TOOL database for dashboard content and analytics" +} + +import { + to = snowflake_account_role.matillion_role_t_role + id = "MATILLION_ROLE__T_ROLE" +} + +resource "snowflake_account_role" "matillion_role_t_role" { + name = "MATILLION_ROLE__T_ROLE" + comment = "Matillion ETL processing service role" +} + +import { + to = snowflake_account_role.omni_infrastructure_automation_t_role + id = "OMNI_INFRASTRUCTURE_AUTOMATION__T_ROLE" +} + +resource "snowflake_account_role" "omni_infrastructure_automation_t_role" { + name = "OMNI_INFRASTRUCTURE_AUTOMATION__T_ROLE" + comment = "Omni infrastructure automation role for automated deployment and management" +} + +import { + to = snowflake_account_role.omni_tech_role_t_role + id = "OMNI_TECH_ROLE__T_ROLE" +} + +resource "snowflake_account_role" "omni_tech_role_t_role" { + name = "OMNI_TECH_ROLE__T_ROLE" + comment = "Omni BI platform technical role - Read access to PROJ_STRIPE, full write access to ANALYTICS_TOOL database" +} + +import { + to = snowflake_account_role.recce_tech_role_t_role + id = "RECCE_TECH_ROLE__T_ROLE" +} + +resource "snowflake_account_role" "recce_tech_role_t_role" { + name = "RECCE_TECH_ROLE__T_ROLE" + comment = "Technical role for Recce dbt validation - Read-only access to PROJ_STRIPE for model validation and data quality testing" +} + +import { + to = snowflake_account_role.sigma_role_t_role + id = "SIGMA_ROLE__T_ROLE" +} + +resource "snowflake_account_role" "sigma_role_t_role" { + name = "SIGMA_ROLE__T_ROLE" + comment = "Sigma BI analytics service role" +} + +import { + to = snowflake_account_role.snowtower_users_t_role + id = "SNOWTOWER_USERS__T_ROLE" +} + +resource "snowflake_account_role" "snowtower_users_t_role" { + name = "SNOWTOWER_USERS__T_ROLE" + comment = "SnowTower application users role" +} + +import { + to = snowflake_account_role.streamlit_role_t_role + id = "STREAMLIT_ROLE__T_ROLE" +} + +resource "snowflake_account_role" "streamlit_role_t_role" { + name = "STREAMLIT_ROLE__T_ROLE" + comment = "Streamlit applications service role" +} + +import { + to = snowflake_account_role.streamlit_towerapp_role_t_role + id = "STREAMLIT_TOWERAPP_ROLE__T_ROLE" +} + +resource "snowflake_account_role" "streamlit_towerapp_role_t_role" { + name = "STREAMLIT_TOWERAPP_ROLE__T_ROLE" + comment = "SnowTower Streamlit application specific role" +} + +import { + to = snowflake_account_role.streamlit_viewer_t_role + id = "STREAMLIT_VIEWER__T_ROLE" +} + +resource "snowflake_account_role" "streamlit_viewer_t_role" { + name = "STREAMLIT_VIEWER__T_ROLE" + comment = "Read-only infrastructure viewer role for Streamlit app users - provides minimal metadata access without data privileges" +} + +import { + to = snowflake_account_role.stripe_t_role + id = "STRIPE__T_ROLE" +} + +resource "snowflake_account_role" "stripe_t_role" { + name = "STRIPE__T_ROLE" + comment = "Stripe data access role with read permissions to SOURCE_STRIPE and write permissions to PROJ_STRIPE" +} + +import { + to = snowflake_account_role.tobiko_cloud_t_role + id = "TOBIKO_CLOUD__T_ROLE" +} + +resource "snowflake_account_role" "tobiko_cloud_t_role" { + name = "TOBIKO_CLOUD__T_ROLE" + comment = "Tobiko Cloud service role" +} diff --git a/terraform-snowflake-boilerplate/users.tf b/terraform-snowflake-boilerplate/users.tf new file mode 100644 index 0000000..1f23372 --- /dev/null +++ b/terraform-snowflake-boilerplate/users.tf @@ -0,0 +1,64 @@ +# Generated by SnowTower - DO NOT EDIT MANUALLY +# Source: snowddl/ YAML configurations + +# User resources + +import { + to = snowflake_user.alice_admin + id = "ALICE_ADMIN" +} + +resource "snowflake_user" "alice_admin" { + name = "ALICE_ADMIN" + email = "alice.admin@example.com" + first_name = "Alice" + last_name = "Admin" + default_role = "SYSADMIN" + comment = "Platform administrator" + user_type = "PERSON" +} + +import { + to = snowflake_user.dbt_service + id = "DBT_SERVICE" +} + +resource "snowflake_user" "dbt_service" { + name = "DBT_SERVICE" + default_role = "DBT_ANALYTICS_ROLE__B_ROLE" + comment = "dbt transformation service account" + user_type = "SERVICE" + rsa_public_key = "MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA0example... +...replace with your actual RSA public key... +AQAB" +} + +import { + to = snowflake_user.dlt_service + id = "DLT_SERVICE" +} + +resource "snowflake_user" "dlt_service" { + name = "DLT_SERVICE" + default_role = "DLT_STRIPE_ROLE__B_ROLE" + comment = "Data ingestion service account" + user_type = "SERVICE" + rsa_public_key = "MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA0example... +...replace with your actual RSA public key... +AQAB" +} + +import { + to = snowflake_user.john_analyst + id = "JOHN_ANALYST" +} + +resource "snowflake_user" "john_analyst" { + name = "JOHN_ANALYST" + email = "john.analyst@example.com" + first_name = "John" + last_name = "Analyst" + default_role = "COMPANY_USERS__B_ROLE" + comment = "Data analyst" + user_type = "PERSON" +} diff --git a/terraform-snowflake-boilerplate/warehouses.tf b/terraform-snowflake-boilerplate/warehouses.tf new file mode 100644 index 0000000..c8419d3 --- /dev/null +++ b/terraform-snowflake-boilerplate/warehouses.tf @@ -0,0 +1,231 @@ +# Generated by SnowTower - DO NOT EDIT MANUALLY +# Source: snowddl/ YAML configurations + +# Warehouse resources + +import { + to = snowflake_warehouse.admin + id = "ADMIN" +} + +resource "snowflake_warehouse" "admin" { + name = "ADMIN" + warehouse_size = "XSMALL" + auto_suspend = 60 + auto_resume = true + comment = "Administrative warehouse for admin operations" +} + +import { + to = snowflake_warehouse.analyst_wh + id = "ANALYST_WH" +} + +resource "snowflake_warehouse" "analyst_wh" { + name = "ANALYST_WH" + warehouse_size = "XSMALL" + auto_suspend = 60 + auto_resume = true + comment = "Warehouse for analyst workloads" + resource_monitor = "ANALYST_WAREHOUSE_MONITOR" +} + +import { + to = snowflake_warehouse.analytics_tool + id = "ANALYTICS_TOOL" +} + +resource "snowflake_warehouse" "analytics_tool" { + name = "ANALYTICS_TOOL" + warehouse_size = "XSMALL" + auto_suspend = 60 + auto_resume = true + comment = "Omni BI platform warehouse for analytics and reporting queries" +} + +import { + to = snowflake_warehouse.bi_tool + id = "BI_TOOL" +} + +resource "snowflake_warehouse" "bi_tool" { + name = "BI_TOOL" + warehouse_size = "XSMALL" + auto_suspend = 60 + auto_resume = true + comment = "LightDash BI platform warehouse for analytics and dashboard queries" +} + +import { + to = snowflake_warehouse.dbt_stripe_wh + id = "DBT_STRIPE_WH" +} + +resource "snowflake_warehouse" "dbt_stripe_wh" { + name = "DBT_STRIPE_WH" + warehouse_size = "XSMALL" + auto_suspend = 60 + auto_resume = true + comment = "dbt Stripe transformation operations" +} + +import { + to = snowflake_warehouse.dev_frank_wh + id = "DEV_FRANK_WH" +} + +resource "snowflake_warehouse" "dev_frank_wh" { + name = "DEV_FRANK_WH" + warehouse_size = "XSMALL" + auto_suspend = 60 + auto_resume = true + comment = "Polo's dedicated development warehouse for data engineering work" +} + +import { + to = snowflake_warehouse.dev_wh + id = "DEV_WH" +} + +resource "snowflake_warehouse" "dev_wh" { + name = "DEV_WH" + warehouse_size = "XSMALL" + auto_suspend = 60 + auto_resume = true + comment = "Development warehouse for personal dev environments" +} + +import { + to = snowflake_warehouse.dlt + id = "DLT" +} + +resource "snowflake_warehouse" "dlt" { + name = "DLT" + warehouse_size = "XSMALL" + auto_suspend = 60 + auto_resume = true + comment = "DLT data loading and pipeline operations" + resource_monitor = "DLT_PIPELINE_MONITOR" +} + +import { + to = snowflake_warehouse.estuary + id = "ESTUARY" +} + +resource "snowflake_warehouse" "estuary" { + name = "ESTUARY" + warehouse_size = "XSMALL" + auto_suspend = 60 + auto_resume = true + comment = "Estuary data integration warehouse" +} + +import { + to = snowflake_warehouse.fabi_ai_warehouse + id = "FABI_AI_WAREHOUSE" +} + +resource "snowflake_warehouse" "fabi_ai_warehouse" { + name = "FABI_AI_WAREHOUSE" + warehouse_size = "XSMALL" + auto_suspend = 60 + auto_resume = true + comment = "Fabi AI workloads and model training" +} + +import { + to = snowflake_warehouse.fivetran + id = "FIVETRAN" +} + +resource "snowflake_warehouse" "fivetran" { + name = "FIVETRAN" + warehouse_size = "XSMALL" + auto_suspend = 60 + auto_resume = true + comment = "Fivetran data integration warehouse" +} + +import { + to = snowflake_warehouse.main_warehouse + id = "MAIN_WAREHOUSE" +} + +resource "snowflake_warehouse" "main_warehouse" { + name = "MAIN_WAREHOUSE" + warehouse_size = "XSMALL" + auto_suspend = 60 + auto_resume = true + comment = "SnowTower team warehouse for data work and analytics" + resource_monitor = "DEV_ENVIRONMENT_MONITOR" +} + +import { + to = snowflake_warehouse.recce + id = "RECCE" +} + +resource "snowflake_warehouse" "recce" { + name = "RECCE" + warehouse_size = "XSMALL" + auto_suspend = 60 + auto_resume = true + comment = "Recce dbt validation and data quality testing warehouse" +} + +import { + to = snowflake_warehouse.streamlit_viewer_wh + id = "STREAMLIT_VIEWER_WH" +} + +resource "snowflake_warehouse" "streamlit_viewer_wh" { + name = "STREAMLIT_VIEWER_WH" + warehouse_size = "XSMALL" + auto_suspend = 60 + auto_resume = true + comment = "Dedicated warehouse for Streamlit viewer metadata queries - minimal resources for cost efficiency" + resource_monitor = "SNOWTOWER_MONITOR" + min_cluster_count = 1 + max_cluster_count = 1 +} + +import { + to = snowflake_warehouse.stripe + id = "STRIPE" +} + +resource "snowflake_warehouse" "stripe" { + name = "STRIPE" + warehouse_size = "XSMALL" + auto_suspend = 60 + auto_resume = true + comment = "Stripe data processing and analytics" +} + +import { + to = snowflake_warehouse.transforming + id = "TRANSFORMING" +} + +resource "snowflake_warehouse" "transforming" { + name = "TRANSFORMING" + warehouse_size = "XSMALL" + auto_suspend = 60 + auto_resume = true + comment = "General data transformation warehouse" +} + +import { + to = snowflake_warehouse.transforming_user + id = "TRANSFORMING_USER" +} + +resource "snowflake_warehouse" "transforming_user" { + name = "TRANSFORMING_USER" + warehouse_size = "XSMALL" + auto_suspend = 60 + auto_resume = true + comment = "Heather's training transformation warehouse" +} diff --git a/tests/integration/test_authentication_methods.py b/tests/integration/test_authentication_methods.py new file mode 100644 index 0000000..c91c3e1 --- /dev/null +++ b/tests/integration/test_authentication_methods.py @@ -0,0 +1,208 @@ +"""Integration tests for authentication methods. + +Tests RSA key operations, Fernet encryption roundtrips, and YAML password handling. +""" + +import os +import subprocess +import tempfile +import pytest +import yaml +from pathlib import Path +from cryptography.fernet import Fernet + +import sys + +sys.path.insert(0, str(Path(__file__).parent.parent.parent / "src")) + +from user_management.encryption import FernetEncryption +from user_management.yaml_handler import YAMLHandler + + +class TestRSAKeyGeneration: + """Test RSA key pair generation and validation.""" + + def test_generate_rsa_key_pair(self): + """Generate an RSA key pair using openssl and verify format.""" + with tempfile.TemporaryDirectory() as tmpdir: + private_key = Path(tmpdir) / "test_key.p8" + public_key = Path(tmpdir) / "test_key.pub" + + # Generate private key + result = subprocess.run( + [ + "bash", + "-c", + f"openssl genrsa 2048 | openssl pkcs8 -topk8 -inform PEM -nocrypt -out {private_key}", + ], + capture_output=True, + text=True, + ) + assert result.returncode == 0, f"Key generation failed: {result.stderr}" + + # Generate public key from private + result = subprocess.run( + [ + "openssl", + "rsa", + "-in", + str(private_key), + "-pubout", + "-out", + str(public_key), + ], + capture_output=True, + text=True, + ) + assert result.returncode == 0 + + # Verify files exist and have content + assert private_key.exists() + assert public_key.exists() + + pub_content = public_key.read_text() + assert "BEGIN PUBLIC KEY" in pub_content + assert "END PUBLIC KEY" in pub_content + + def test_rsa_public_key_in_yaml_format(self): + """Verify RSA public key can be stored in YAML user config format.""" + with tempfile.TemporaryDirectory() as tmpdir: + private_key = Path(tmpdir) / "test_key.p8" + public_key = Path(tmpdir) / "test_key.pub" + + subprocess.run( + [ + "bash", + "-c", + f"openssl genrsa 2048 | openssl pkcs8 -topk8 -inform PEM -nocrypt -out {private_key}", + ], + capture_output=True, + ) + subprocess.run( + ["openssl", "rsa", "-in", str(private_key), "-pubout", "-out", str(public_key)], + capture_output=True, + ) + + pub_content = public_key.read_text() + # Strip headers for SnowDDL format + lines = pub_content.strip().split("\n") + key_body = "\n".join(lines[1:-1]) + + user_config = { + "TEST_SERVICE": { + "type": "SERVICE", + "rsa_public_key": key_body, + "default_role": "TEST_ROLE", + } + } + + yaml_output = yaml.dump(user_config, default_flow_style=False) + reloaded = yaml.safe_load(yaml_output) + + assert reloaded["TEST_SERVICE"]["rsa_public_key"] == key_body + + +class TestFernetEncryptionRoundtrip: + """Test full encryption/decryption lifecycle.""" + + def test_encrypt_decrypt_roundtrip(self): + """Encrypt a password and decrypt it back.""" + key = Fernet.generate_key().decode("utf-8") + enc = FernetEncryption(key=key) + + passwords = [ + "SimplePassword123!", + "P@$$w0rd!#%^&*()", + "Unicode: Ünîcödé 你好", + "", + "A" * 1000, + ] + + for password in passwords: + encrypted = enc.encrypt_password(password) + decrypted = enc.decrypt_password(encrypted) + assert decrypted == password, f"Roundtrip failed for: {password!r}" + + def test_key_rotation_preserves_passwords(self): + """Key rotation should re-encrypt without losing data.""" + old_key = Fernet.generate_key().decode("utf-8") + new_key = Fernet.generate_key().decode("utf-8") + + old_enc = FernetEncryption(key=old_key) + new_enc = FernetEncryption(key=new_key) + + original_passwords = { + "user1": "Password1!", + "user2": "Password2!", + "user3": "Password3!", + } + + encrypted = {user: old_enc.encrypt_password(pw) for user, pw in original_passwords.items()} + + rotated = old_enc.rotate_keys(old_key, new_key, encrypted) + + for user, original_pw in original_passwords.items(): + decrypted = new_enc.decrypt_password(rotated[user]) + assert decrypted == original_pw + + def test_wrong_key_fails(self): + """Decryption with wrong key should fail cleanly.""" + key1 = Fernet.generate_key().decode("utf-8") + key2 = Fernet.generate_key().decode("utf-8") + + enc1 = FernetEncryption(key=key1) + enc2 = FernetEncryption(key=key2) + + encrypted = enc1.encrypt_password("Secret123!") + + from user_management.encryption import InvalidEncryptedDataError + + with pytest.raises(InvalidEncryptedDataError): + enc2.decrypt_password(encrypted) + + +class TestYAMLPasswordHandling: + """Test encrypted passwords in YAML config files.""" + + def test_user_yaml_with_encrypted_password(self): + """Users with encrypted passwords should load and save correctly.""" + with tempfile.TemporaryDirectory() as tmpdir: + config_dir = Path(tmpdir) + handler = YAMLHandler(config_directory=config_dir) + + users = { + "ENCRYPTED_USER": { + "type": "PERSON", + "login_name": "ENCRYPTED_USER", + "email": "test@example.com", + "password": "!decrypt gAAAAABencryptedplaceholder", + } + } + + handler.save_users(users, backup=False) + loaded = handler.load_users() + + assert "ENCRYPTED_USER" in loaded + assert "password" in loaded["ENCRYPTED_USER"] + + def test_service_account_no_password(self): + """Service accounts should work without passwords.""" + with tempfile.TemporaryDirectory() as tmpdir: + config_dir = Path(tmpdir) + handler = YAMLHandler(config_directory=config_dir) + + users = { + "SVC_ACCOUNT": { + "type": "SERVICE", + "login_name": "SVC_ACCOUNT", + "rsa_public_key": "MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA...", + "default_role": "SERVICE_ROLE", + } + } + + handler.save_users(users, backup=False) + loaded = handler.load_users() + + assert "SVC_ACCOUNT" in loaded + assert "password" not in loaded["SVC_ACCOUNT"] + assert loaded["SVC_ACCOUNT"]["rsa_public_key"].startswith("MIIBIjAN") diff --git a/tests/integration/test_cli_commands.py b/tests/integration/test_cli_commands.py new file mode 100644 index 0000000..c1b0063 --- /dev/null +++ b/tests/integration/test_cli_commands.py @@ -0,0 +1,123 @@ +"""Integration tests for CLI commands. + +Tests that all registered UV commands are accessible and respond correctly. +""" + +import subprocess +import pytest +from pathlib import Path + +PROJECT_ROOT = Path(__file__).parent.parent.parent + + +def run_command(cmd, timeout=30): + """Run a UV command and return the result.""" + return subprocess.run( + ["uv", "run"] + cmd, + capture_output=True, + text=True, + cwd=PROJECT_ROOT, + timeout=timeout, + ) + + +class TestCoreCommands: + """Test core SnowDDL operation commands.""" + + def test_snowddl_plan_runs(self): + """snowddl-plan should execute (may fail on missing connection).""" + result = run_command(["snowddl-plan"]) + # Should not be 'command not found' (127) + assert result.returncode != 127 + + def test_snowddl_apply_has_help(self): + """snowddl-apply should be a registered command.""" + result = run_command(["snowddl-apply", "--help"]) + assert result.returncode != 127 + + def test_snowddl_validate_exists(self): + """snowddl-validate should be a registered command.""" + result = run_command(["snowddl-validate", "--help"]) + assert result.returncode != 127 + + def test_snowddl_diff_exists(self): + """snowddl-diff should be a registered command.""" + result = run_command(["snowddl-diff", "--help"]) + assert result.returncode != 127 + + +class TestUserManagementCommands: + """Test user management commands.""" + + def test_manage_users_help(self): + """manage-users --help should show usage.""" + result = run_command(["manage-users", "--help"]) + assert result.returncode == 0 + assert "usage" in result.stdout.lower() or "manage" in result.stdout.lower() + + def test_util_generate_key(self): + """util-generate-key should generate a Fernet key.""" + result = run_command(["util-generate-key"]) + # Should output a base64-encoded key or help text + assert result.returncode != 127 + + +class TestResourceManagementCommands: + """Test resource management commands.""" + + def test_manage_warehouses_help(self): + """manage-warehouses --help should show usage.""" + result = run_command(["manage-warehouses", "--help"]) + assert result.returncode == 0 + + def test_manage_costs_help(self): + """manage-costs --help should show usage.""" + result = run_command(["manage-costs", "--help"]) + assert result.returncode == 0 + + def test_manage_security_help(self): + """manage-security --help should show usage.""" + result = run_command(["manage-security", "--help"]) + assert result.returncode == 0 + + def test_manage_backup_help(self): + """manage-backup --help should show usage.""" + result = run_command(["manage-backup", "--help"]) + assert result.returncode == 0 + + +class TestMonitoringCommands: + """Test monitoring commands.""" + + def test_monitor_health_help(self): + """monitor-health --help should show usage.""" + result = run_command(["monitor-health", "--help"]) + assert result.returncode == 0 + + def test_monitor_audit_help(self): + """monitor-audit --help should show usage.""" + result = run_command(["monitor-audit", "--help"]) + assert result.returncode == 0 + + def test_monitor_metrics_help(self): + """monitor-metrics --help should show usage.""" + result = run_command(["monitor-metrics", "--help"]) + assert result.returncode == 0 + + +class TestHelpCommand: + """Test help and discovery.""" + + def test_snowtower_help(self): + """snowtower should show available commands.""" + result = run_command(["snowtower"]) + assert result.returncode == 0 + + +class TestDeploySafe: + """Test deploy-safe command.""" + + def test_deploy_safe_help(self): + """deploy-safe --help should show usage.""" + result = run_command(["deploy-safe", "--help"]) + assert result.returncode == 0 diff --git a/tests/integration/test_deployment_workflow.py b/tests/integration/test_deployment_workflow.py new file mode 100644 index 0000000..4eb7c76 --- /dev/null +++ b/tests/integration/test_deployment_workflow.py @@ -0,0 +1,203 @@ +"""Integration tests for deployment workflow. + +Tests the plan -> review -> apply lifecycle with mocked Snowflake connections. +""" + +import subprocess +import tempfile +import shutil +import pytest +import yaml +from pathlib import Path +from unittest.mock import patch, Mock + +import sys + +sys.path.insert(0, str(Path(__file__).parent.parent.parent / "src")) + +PROJECT_ROOT = Path(__file__).parent.parent.parent + + +class TestPlanWorkflow: + """Test the snowddl-plan command behavior.""" + + def test_plan_with_valid_config(self): + """Plan should run against the real snowddl/ directory.""" + result = subprocess.run( + ["uv", "run", "snowddl-plan"], + capture_output=True, + text=True, + cwd=PROJECT_ROOT, + timeout=30, + ) + # May fail due to missing Snowflake connection, but shouldn't crash + assert result.returncode != 127, "snowddl-plan command not found" + + def test_plan_output_format(self): + """Plan output should contain recognizable SnowDDL markers.""" + result = subprocess.run( + ["uv", "run", "snowddl-plan"], + capture_output=True, + text=True, + cwd=PROJECT_ROOT, + timeout=30, + ) + output = result.stdout + result.stderr + # Should have some output, even if it's an error message + assert len(output) > 0 + + +class TestDeploySafeWorkflow: + """Test the deploy-safe wrapper.""" + + def test_deploy_safe_exists(self): + """deploy-safe command should be available.""" + result = subprocess.run( + ["uv", "run", "deploy-safe", "--help"], + capture_output=True, + text=True, + cwd=PROJECT_ROOT, + timeout=30, + ) + assert result.returncode != 127 + + def test_deploy_safe_has_dry_run(self): + """deploy-safe should support --dry-run or similar safety flag.""" + result = subprocess.run( + ["uv", "run", "deploy-safe", "--help"], + capture_output=True, + text=True, + cwd=PROJECT_ROOT, + timeout=30, + ) + # Check help text exists + assert len(result.stdout + result.stderr) > 0 + + +class TestYAMLConfigLifecycle: + """Test adding/modifying/removing users through YAML.""" + + def setup_method(self): + """Create a temporary snowddl config directory.""" + self.tmpdir = tempfile.mkdtemp() + self.config_dir = Path(self.tmpdir) + + def teardown_method(self): + """Clean up temp directory.""" + shutil.rmtree(self.tmpdir, ignore_errors=True) + + def test_add_user_to_yaml(self): + """Adding a user to YAML should be readable back.""" + from user_management.yaml_handler import YAMLHandler + + handler = YAMLHandler(config_directory=self.config_dir) + + user_data = { + "NEW_ANALYST": { + "type": "PERSON", + "login_name": "NEW_ANALYST", + "email": "analyst@example.com", + "first_name": "New", + "last_name": "Analyst", + "default_role": "ANALYST_ROLE", + } + } + + handler.save_users(user_data, backup=False) + loaded = handler.load_users() + + assert "NEW_ANALYST" in loaded + assert loaded["NEW_ANALYST"]["email"] == "analyst@example.com" + + def test_modify_user_in_yaml(self): + """Modifying a user should persist changes.""" + from user_management.yaml_handler import YAMLHandler + + handler = YAMLHandler(config_directory=self.config_dir) + + # Create initial user + handler.save_users( + {"TEST_USER": {"type": "PERSON", "login_name": "TEST_USER", "email": "old@example.com"}}, + backup=False, + ) + + # Load, modify, save + users = handler.load_users() + users["TEST_USER"]["email"] = "new@example.com" + handler.save_users(users, backup=False) + + # Verify + reloaded = handler.load_users() + assert reloaded["TEST_USER"]["email"] == "new@example.com" + + def test_remove_user_from_yaml(self): + """Removing a user should persist.""" + from user_management.yaml_handler import YAMLHandler + + handler = YAMLHandler(config_directory=self.config_dir) + + handler.save_users( + { + "USER_A": {"type": "PERSON", "login_name": "USER_A"}, + "USER_B": {"type": "PERSON", "login_name": "USER_B"}, + }, + backup=False, + ) + + users = handler.load_users() + del users["USER_A"] + handler.save_users(users, backup=False) + + reloaded = handler.load_users() + assert "USER_A" not in reloaded + assert "USER_B" in reloaded + + def test_backup_before_modification(self): + """Backup should be created before modifications.""" + from user_management.yaml_handler import YAMLHandler + + handler = YAMLHandler(config_directory=self.config_dir) + + # Save initial state + handler.save_users({"ORIGINAL": {"type": "PERSON", "login_name": "ORIGINAL"}}, backup=False) + + # Modify with backup + handler.save_users({"MODIFIED": {"type": "PERSON", "login_name": "MODIFIED"}}, backup=True) + + # Check backup exists + backups = list(handler.backup_dir.glob("user_*.yaml")) + assert len(backups) >= 1 + + # Verify backup contains original data + with open(backups[0]) as f: + backup_data = yaml.safe_load(f) + assert "ORIGINAL" in backup_data + + +class TestRollbackWorkflow: + """Test rollback capabilities.""" + + def test_git_rollback_of_yaml(self): + """Verify git can rollback YAML changes (simulated).""" + with tempfile.TemporaryDirectory() as tmpdir: + config_dir = Path(tmpdir) + user_yaml = config_dir / "user.yaml" + + # Write v1 + v1 = {"USER_V1": {"type": "PERSON"}} + user_yaml.write_text(yaml.dump(v1)) + + # Write v2 + v2 = {"USER_V2": {"type": "SERVICE"}} + user_yaml.write_text(yaml.dump(v2)) + + # Verify v2 is current + current = yaml.safe_load(user_yaml.read_text()) + assert "USER_V2" in current + assert "USER_V1" not in current + + # Simulate rollback by restoring v1 + user_yaml.write_text(yaml.dump(v1)) + restored = yaml.safe_load(user_yaml.read_text()) + assert "USER_V1" in restored + assert "USER_V2" not in restored diff --git a/tests/integration/test_yaml_validation.py b/tests/integration/test_yaml_validation.py new file mode 100644 index 0000000..2992922 --- /dev/null +++ b/tests/integration/test_yaml_validation.py @@ -0,0 +1,274 @@ +"""Integration tests for YAML configuration validation. + +Tests that the actual snowddl/ YAML files are well-formed and internally consistent. +""" + +import pytest +import yaml +from pathlib import Path + +SNOWDDL_DIR = Path(__file__).parent.parent.parent / "snowddl" + +SYSTEM_ROLES = { + "ACCOUNTADMIN", + "SECURITYADMIN", + "USERADMIN", + "SYSADMIN", + "PUBLIC", + "ORGADMIN", +} + +VALID_WAREHOUSE_SIZES = { + "X-Small", + "Small", + "Medium", + "Large", + "X-Large", + "2X-Large", + "3X-Large", + "4X-Large", + "5X-Large", + "6X-Large", +} + +VALID_OBJECT_TYPES = { + "DATABASE", + "SCHEMA", + "TABLE", + "VIEW", + "WAREHOUSE", + "STAGE", + "FUNCTION", + "PROCEDURE", + "FILE_FORMAT", + "SEQUENCE", +} + + +def load_yaml(filename): + """Load a YAML file from snowddl/ with !decrypt tag support.""" + + def decrypt_constructor(loader, node): + return "!decrypt " + loader.construct_scalar(node) + + loader = yaml.SafeLoader + loader.add_constructor("!decrypt", decrypt_constructor) + filepath = SNOWDDL_DIR / filename + if not filepath.exists(): + return {} + with open(filepath) as f: + data = yaml.load(f, Loader=loader) + return data or {} + + +class TestYAMLSyntax: + """Test that all YAML files parse without errors.""" + + @pytest.mark.parametrize( + "filename", + [ + "user.yaml", + "warehouse.yaml", + "business_role.yaml", + "tech_role.yaml", + "network_policy.yaml", + "authentication_policy.yaml", + "password_policy.yaml", + "session_policy.yaml", + "resource_monitor.yaml", + ], + ) + def test_yaml_parses(self, filename): + """Each YAML config file should parse without errors.""" + data = load_yaml(filename) + assert isinstance(data, dict), f"{filename} should contain a dictionary" + + +class TestUserConfig: + """Validate user.yaml configuration.""" + + def test_all_users_have_type(self): + """Every user must have a type field.""" + users = load_yaml("user.yaml") + for username, config in users.items(): + assert "type" in config, f"User {username} missing 'type' field" + assert config["type"] in ( + "PERSON", + "SERVICE", + ), f"User {username} has invalid type: {config['type']}" + + def test_person_users_have_email(self): + """PERSON users should have an email.""" + users = load_yaml("user.yaml") + for username, config in users.items(): + if config.get("type") == "PERSON": + assert "email" in config, f"PERSON user {username} missing 'email'" + + def test_service_users_have_rsa_key(self): + """SERVICE users should have an RSA public key.""" + users = load_yaml("user.yaml") + for username, config in users.items(): + if config.get("type") == "SERVICE": + assert ( + "rsa_public_key" in config + ), f"SERVICE user {username} missing 'rsa_public_key'" + + def test_no_duplicate_users(self): + """No duplicate user names (YAML handles this natively, but verify).""" + users = load_yaml("user.yaml") + assert len(users) > 0, "user.yaml should have at least one user" + + +class TestWarehouseConfig: + """Validate warehouse.yaml configuration.""" + + def test_all_warehouses_have_size(self): + """Every warehouse must have a size.""" + warehouses = load_yaml("warehouse.yaml") + for wh_name, config in warehouses.items(): + assert "size" in config, f"Warehouse {wh_name} missing 'size'" + + def test_warehouse_sizes_are_valid(self): + """Warehouse sizes must be valid Snowflake sizes.""" + warehouses = load_yaml("warehouse.yaml") + for wh_name, config in warehouses.items(): + assert ( + config["size"] in VALID_WAREHOUSE_SIZES + ), f"Warehouse {wh_name} has invalid size: {config['size']}" + + def test_auto_suspend_is_positive(self): + """auto_suspend should be a positive integer.""" + warehouses = load_yaml("warehouse.yaml") + for wh_name, config in warehouses.items(): + if "auto_suspend" in config: + assert ( + isinstance(config["auto_suspend"], int) and config["auto_suspend"] > 0 + ), f"Warehouse {wh_name} has invalid auto_suspend: {config['auto_suspend']}" + + +class TestBusinessRoleConfig: + """Validate business_role.yaml configuration.""" + + def test_tech_roles_exist(self): + """All tech_roles referenced in business roles should exist in tech_role.yaml.""" + business_roles = load_yaml("business_role.yaml") + tech_roles = load_yaml("tech_role.yaml") + + for role_name, config in business_roles.items(): + for tech_role in config.get("tech_roles", []): + assert ( + tech_role in tech_roles + ), f"Business role {role_name} references missing tech role: {tech_role}" + + def test_warehouse_usage_exists(self): + """All warehouse_usage entries should reference existing warehouses.""" + business_roles = load_yaml("business_role.yaml") + warehouses = load_yaml("warehouse.yaml") + + for role_name, config in business_roles.items(): + for wh in config.get("warehouse_usage", []): + assert ( + wh in warehouses + ), f"Business role {role_name} references missing warehouse: {wh}" + + def test_schema_owner_format(self): + """schema_owner entries should be DB.SCHEMA format.""" + business_roles = load_yaml("business_role.yaml") + + for role_name, config in business_roles.items(): + for entry in config.get("schema_owner", []): + parts = entry.split(".") + assert ( + len(parts) == 2 + ), f"Business role {role_name} schema_owner '{entry}' not in DB.SCHEMA format" + + +class TestTechRoleConfig: + """Validate tech_role.yaml configuration.""" + + def test_grant_keys_are_valid(self): + """Grant keys must follow OBJECT_TYPE:PRIVILEGE format.""" + tech_roles = load_yaml("tech_role.yaml") + + for role_name, config in tech_roles.items(): + for grant_key in config.get("grants", {}): + parts = grant_key.split(":", 1) + assert ( + len(parts) == 2 + ), f"Tech role {role_name} grant key '{grant_key}' not in TYPE:PRIVILEGE format" + assert ( + parts[0] in VALID_OBJECT_TYPES + ), f"Tech role {role_name} has invalid object type: {parts[0]}" + + for grant_key in config.get("future_grants", {}): + parts = grant_key.split(":", 1) + assert ( + len(parts) == 2 + ), f"Tech role {role_name} future_grant key '{grant_key}' not in TYPE:PRIVILEGE format" + assert ( + parts[0] in VALID_OBJECT_TYPES + ), f"Tech role {role_name} future_grant has invalid object type: {parts[0]}" + + def test_warehouse_grants_reference_existing(self): + """WAREHOUSE:USAGE grants should reference existing warehouses.""" + tech_roles = load_yaml("tech_role.yaml") + warehouses = load_yaml("warehouse.yaml") + + for role_name, config in tech_roles.items(): + for grant_key, targets in config.get("grants", {}).items(): + if grant_key.startswith("WAREHOUSE:"): + for wh in targets or []: + assert ( + wh in warehouses + ), f"Tech role {role_name} references missing warehouse: {wh}" + + +class TestResourceMonitorConfig: + """Validate resource_monitor.yaml configuration.""" + + def test_monitors_have_credit_quota(self): + """Every monitor must have a credit_quota.""" + monitors = load_yaml("resource_monitor.yaml") + for name, config in monitors.items(): + assert "credit_quota" in config, f"Monitor {name} missing 'credit_quota'" + assert isinstance( + config["credit_quota"], (int, float) + ), f"Monitor {name} credit_quota must be numeric" + + def test_warehouse_monitors_exist(self): + """resource_monitor references in warehouse.yaml should exist.""" + warehouses = load_yaml("warehouse.yaml") + monitors = load_yaml("resource_monitor.yaml") + + for wh_name, config in warehouses.items(): + if "resource_monitor" in config: + assert ( + config["resource_monitor"] in monitors + ), f"Warehouse {wh_name} references missing monitor: {config['resource_monitor']}" + + +class TestNetworkPolicyConfig: + """Validate network_policy.yaml configuration.""" + + def test_policies_have_allowed_ips(self): + """Every network policy should have allowed_ip_list.""" + policies = load_yaml("network_policy.yaml") + for name, config in policies.items(): + assert ( + "allowed_ip_list" in config + ), f"Network policy {name} missing 'allowed_ip_list'" + assert isinstance( + config["allowed_ip_list"], list + ), f"Network policy {name} allowed_ip_list must be a list" + + def test_ip_addresses_are_cidr(self): + """IP addresses should be in CIDR notation.""" + import ipaddress + + policies = load_yaml("network_policy.yaml") + for name, config in policies.items(): + for ip in config.get("allowed_ip_list", []): + try: + ipaddress.ip_network(ip, strict=False) + except ValueError: + pytest.fail(f"Network policy {name} has invalid CIDR: {ip}") diff --git a/tests/test_generate_terraform.py b/tests/test_generate_terraform.py new file mode 100644 index 0000000..b2f160a --- /dev/null +++ b/tests/test_generate_terraform.py @@ -0,0 +1,703 @@ +""" +Test Suite for Terraform HCL Generator from SnowDDL YAML. + +Tests the generate_terraform.py script including: +- Name conversion helpers (to_tf_name, hcl_value, hcl_block) +- YAML loading with !decrypt tag handling +- Individual generators (users, warehouses, roles, policies, etc.) +- End-to-end generate_all and write_to_directory +- Output rendering helpers +""" + +import sys +from pathlib import Path + +# Make scripts/ importable +sys.path.insert(0, str(Path(__file__).parent.parent / "scripts")) + +import pytest +import yaml + +from generate_terraform import ( + to_tf_name, + hcl_value, + hcl_block, + load_yaml, + generate_users, + generate_warehouses, + generate_business_roles, + generate_tech_roles, + generate_network_policies, + generate_resource_monitors, + generate_databases, + generate_all, + write_to_directory, + render_section, + TerraformOutput, + HEADER, +) + + +# --------------------------------------------------------------------------- +# to_tf_name +# --------------------------------------------------------------------------- + + +class TestToTfName: + """Tests for the to_tf_name name conversion helper.""" + + def test_uppercase_to_lowercase(self): + assert to_tf_name("MY_WAREHOUSE") == "my_warehouse" + + def test_hyphens_to_underscores(self): + assert to_tf_name("my-warehouse") == "my_warehouse" + + def test_special_characters_replaced(self): + assert to_tf_name("WH@#$NAME") == "wh_name" + + def test_consecutive_underscores_collapsed(self): + assert to_tf_name("WH___NAME") == "wh_name" + + def test_leading_trailing_underscores_stripped(self): + assert to_tf_name("__WH__") == "wh" + + def test_dots_replaced(self): + assert to_tf_name("DB.SCHEMA") == "db_schema" + + def test_mixed_case_and_special(self): + assert to_tf_name("My-Cool_Role__B_ROLE") == "my_cool_role_b_role" + + def test_already_valid(self): + assert to_tf_name("simple_name") == "simple_name" + + def test_single_char(self): + assert to_tf_name("X") == "x" + + def test_numbers_preserved(self): + assert to_tf_name("WH_2X_LARGE") == "wh_2x_large" + + +# --------------------------------------------------------------------------- +# hcl_value +# --------------------------------------------------------------------------- + + +class TestHclValue: + """Tests for the hcl_value formatting function.""" + + def test_bool_true(self): + assert hcl_value(True) == "true" + + def test_bool_false(self): + assert hcl_value(False) == "false" + + def test_int(self): + assert hcl_value(42) == "42" + + def test_float(self): + assert hcl_value(3.14) == "3.14" + + def test_string(self): + assert hcl_value("hello") == '"hello"' + + def test_string_with_quotes(self): + assert hcl_value('say "hi"') == '"say \\"hi\\""' + + def test_string_with_backslash(self): + assert hcl_value("back\\slash") == '"back\\\\slash"' + + def test_list_of_strings(self): + assert hcl_value(["a", "b"]) == '["a", "b"]' + + def test_list_of_ints(self): + assert hcl_value([1, 2, 3]) == "[1, 2, 3]" + + def test_empty_list(self): + assert hcl_value([]) == "[]" + + def test_fallback_type(self): + """Non-standard types are stringified.""" + result = hcl_value(None) + assert result == '"None"' + + +# --------------------------------------------------------------------------- +# hcl_block +# --------------------------------------------------------------------------- + + +class TestHclBlock: + """Tests for the hcl_block rendering function.""" + + def test_basic_block(self): + result = hcl_block("snowflake_user", "alice", {"name": "ALICE"}) + assert 'resource "snowflake_user" "alice"' in result + assert 'name = "ALICE"' in result + assert result.endswith("}") + + def test_block_with_import_id(self): + result = hcl_block("snowflake_user", "alice", {"name": "ALICE"}, import_id="ALICE") + assert "import {" in result + assert "to = snowflake_user.alice" in result + assert 'id = "ALICE"' in result + + def test_block_without_import_id(self): + result = hcl_block("snowflake_user", "alice", {"name": "ALICE"}) + assert "import" not in result + + def test_nested_block(self): + """Nested dict values render as HCL sub-blocks.""" + attrs = { + "name": "WH", + "on_account_object": {"object_type": "WAREHOUSE", "object_name": "WH"}, + } + result = hcl_block("snowflake_grant", "g", attrs) + assert "on_account_object {" in result + assert "object_type" in result + + def test_bool_and_int_values(self): + attrs = {"auto_resume": True, "auto_suspend": 120} + result = hcl_block("snowflake_warehouse", "wh", attrs) + assert "true" in result + assert "120" in result + + def test_empty_attrs(self): + """A block with no attributes still renders valid HCL.""" + result = hcl_block("snowflake_user", "empty", {}) + assert 'resource "snowflake_user" "empty" {' in result + assert result.strip().endswith("}") + + +# --------------------------------------------------------------------------- +# load_yaml +# --------------------------------------------------------------------------- + + +class TestLoadYaml: + """Tests for the load_yaml helper.""" + + def test_valid_file(self, tmp_path): + p = tmp_path / "test.yaml" + p.write_text(yaml.dump({"KEY": "value"})) + data = load_yaml(p) + assert data == {"KEY": "value"} + + def test_missing_file(self, tmp_path): + data = load_yaml(tmp_path / "missing.yaml") + assert data is None + + def test_empty_file(self, tmp_path): + p = tmp_path / "empty.yaml" + p.write_text("") + data = load_yaml(p) + assert data is None # None because safe_load returns None, not dict + + def test_non_dict_file(self, tmp_path): + p = tmp_path / "list.yaml" + p.write_text("- item1\n- item2\n") + data = load_yaml(p) + assert data is None # list is not dict + + def test_decrypt_tag_handled(self, tmp_path): + """The !decrypt tag should not raise an error and returns a placeholder.""" + p = tmp_path / "encrypted.yaml" + p.write_text('USER:\n password: !decrypt "gAAAAABf..."\n') + data = load_yaml(p) + assert data is not None + assert data["USER"]["password"] == "" + + +# --------------------------------------------------------------------------- +# generate_users +# --------------------------------------------------------------------------- + + +class TestGenerateUsers: + """Tests for generate_users.""" + + def test_generates_user_blocks(self, tmp_path): + data = { + "ALICE": { + "type": "PERSON", + "email": "alice@example.com", + "first_name": "Alice", + "last_name": "Smith", + "default_role": "ANALYST", + "comment": "Test user", + } + } + (tmp_path / "user.yaml").write_text(yaml.dump(data)) + blocks = generate_users(tmp_path) + assert len(blocks) == 1 + assert "snowflake_user" in blocks[0] + assert '"ALICE"' in blocks[0] + assert '"alice@example.com"' in blocks[0] + assert "user_type" in blocks[0] # 'type' maps to 'user_type' + + def test_rsa_key_included(self, tmp_path): + data = { + "SVC": { + "type": "SERVICE", + "rsa_public_key": "MIIBIjANBg...", + } + } + (tmp_path / "user.yaml").write_text(yaml.dump(data)) + blocks = generate_users(tmp_path) + assert len(blocks) == 1 + assert "rsa_public_key" in blocks[0] + + def test_empty_user_yaml(self, tmp_path): + (tmp_path / "user.yaml").write_text("") + blocks = generate_users(tmp_path) + assert blocks == [] + + def test_no_user_yaml(self, tmp_path): + blocks = generate_users(tmp_path) + assert blocks == [] + + def test_multiple_users_sorted(self, tmp_path): + data = { + "ZARA": {"type": "PERSON"}, + "ALICE": {"type": "PERSON"}, + } + (tmp_path / "user.yaml").write_text(yaml.dump(data)) + blocks = generate_users(tmp_path) + assert len(blocks) == 2 + # ALICE should come first (sorted) + assert "ALICE" in blocks[0] + assert "ZARA" in blocks[1] + + def test_import_block_present(self, tmp_path): + data = {"BOB": {"type": "PERSON"}} + (tmp_path / "user.yaml").write_text(yaml.dump(data)) + blocks = generate_users(tmp_path) + assert "import {" in blocks[0] + assert 'id = "BOB"' in blocks[0] + + +# --------------------------------------------------------------------------- +# generate_warehouses +# --------------------------------------------------------------------------- + + +class TestGenerateWarehouses: + """Tests for generate_warehouses.""" + + def test_generates_warehouse_block(self, tmp_path): + data = { + "COMPUTE_WH": { + "size": "X-Small", + "auto_suspend": 120, + "auto_resume": True, + "comment": "Main warehouse", + } + } + (tmp_path / "warehouse.yaml").write_text(yaml.dump(data)) + blocks = generate_warehouses(tmp_path) + assert len(blocks) == 1 + assert "snowflake_warehouse" in blocks[0] + assert '"COMPUTE_WH"' in blocks[0] + assert "XSMALL" in blocks[0] # size mapped + assert "120" in blocks[0] + + def test_size_mapping(self, tmp_path): + """Various size formats are correctly mapped.""" + for yaml_size, expected_tf in [("Small", "SMALL"), ("X-Large", "XLARGE"), ("2X-Large", "XXLARGE")]: + data = {"WH": {"size": yaml_size}} + (tmp_path / "warehouse.yaml").write_text(yaml.dump(data)) + blocks = generate_warehouses(tmp_path) + assert expected_tf in blocks[0], f"Expected {expected_tf} for size {yaml_size}" + + def test_default_auto_resume(self, tmp_path): + """auto_resume defaults to True if not specified.""" + data = {"WH": {"size": "Small"}} + (tmp_path / "warehouse.yaml").write_text(yaml.dump(data)) + blocks = generate_warehouses(tmp_path) + assert "auto_resume" in blocks[0] + assert "true" in blocks[0] + + def test_empty_warehouse_yaml(self, tmp_path): + (tmp_path / "warehouse.yaml").write_text("") + blocks = generate_warehouses(tmp_path) + assert blocks == [] + + def test_resource_monitor_included(self, tmp_path): + data = {"WH": {"size": "Small", "resource_monitor": "MY_MON"}} + (tmp_path / "warehouse.yaml").write_text(yaml.dump(data)) + blocks = generate_warehouses(tmp_path) + assert "resource_monitor" in blocks[0] + assert "MY_MON" in blocks[0] + + +# --------------------------------------------------------------------------- +# generate_business_roles +# --------------------------------------------------------------------------- + + +class TestGenerateBusinessRoles: + """Tests for generate_business_roles.""" + + def test_generates_role_and_grant_blocks(self, tmp_path): + data = { + "ANALYST": { + "tech_roles": ["READ_ROLE"], + "warehouse_usage": ["COMPUTE_WH"], + "comment": "Analyst role", + } + } + (tmp_path / "business_role.yaml").write_text(yaml.dump(data)) + role_blocks, grant_blocks = generate_business_roles(tmp_path) + assert len(role_blocks) == 1 + assert "ANALYST__B_ROLE" in role_blocks[0] + # tech role inheritance grant + warehouse usage grant + assert len(grant_blocks) == 2 + + def test_schema_owner_grant(self, tmp_path): + data = { + "ANALYST": { + "schema_owner": ["MY_DB.MY_SCHEMA"], + } + } + (tmp_path / "business_role.yaml").write_text(yaml.dump(data)) + role_blocks, grant_blocks = generate_business_roles(tmp_path) + assert len(grant_blocks) == 1 + assert "OWNER__S_ROLE" in grant_blocks[0] + + def test_empty_business_role_yaml(self, tmp_path): + (tmp_path / "business_role.yaml").write_text("") + role_blocks, grant_blocks = generate_business_roles(tmp_path) + assert role_blocks == [] + assert grant_blocks == [] + + def test_no_file(self, tmp_path): + role_blocks, grant_blocks = generate_business_roles(tmp_path) + assert role_blocks == [] + assert grant_blocks == [] + + def test_role_import_block(self, tmp_path): + data = {"ANALYST": {"comment": "test"}} + (tmp_path / "business_role.yaml").write_text(yaml.dump(data)) + role_blocks, _ = generate_business_roles(tmp_path) + assert "import {" in role_blocks[0] + assert "ANALYST__B_ROLE" in role_blocks[0] + + +# --------------------------------------------------------------------------- +# generate_tech_roles +# --------------------------------------------------------------------------- + + +class TestGenerateTechRoles: + """Tests for generate_tech_roles.""" + + def test_generates_role_and_grants(self, tmp_path): + data = { + "READ_ROLE": { + "grants": { + "DATABASE:USAGE": ["MY_DB"], + "SCHEMA:USAGE": ["MY_DB.PUBLIC"], + }, + } + } + (tmp_path / "tech_role.yaml").write_text(yaml.dump(data)) + role_blocks, grant_blocks = generate_tech_roles(tmp_path) + assert len(role_blocks) == 1 + assert "READ_ROLE__T_ROLE" in role_blocks[0] + assert len(grant_blocks) == 2 # one for DATABASE, one for SCHEMA + + def test_future_grants(self, tmp_path): + data = { + "READ_ROLE": { + "future_grants": { + "TABLE:SELECT": ["MY_DB"], + }, + } + } + (tmp_path / "tech_role.yaml").write_text(yaml.dump(data)) + role_blocks, grant_blocks = generate_tech_roles(tmp_path) + assert len(grant_blocks) == 1 + assert "future" in grant_blocks[0].lower() or "TABLES" in grant_blocks[0] + + def test_schema_grant_has_on_schema(self, tmp_path): + data = { + "ROLE": { + "grants": {"SCHEMA:USAGE": ["DB.SCHEMA"]}, + } + } + (tmp_path / "tech_role.yaml").write_text(yaml.dump(data)) + _, grant_blocks = generate_tech_roles(tmp_path) + assert "on_schema" in grant_blocks[0] + + def test_warehouse_grant_has_on_account_object(self, tmp_path): + data = { + "ROLE": { + "grants": {"WAREHOUSE:USAGE": ["WH"]}, + } + } + (tmp_path / "tech_role.yaml").write_text(yaml.dump(data)) + _, grant_blocks = generate_tech_roles(tmp_path) + assert "on_account_object" in grant_blocks[0] + + def test_empty_tech_role_yaml(self, tmp_path): + (tmp_path / "tech_role.yaml").write_text("") + role_blocks, grant_blocks = generate_tech_roles(tmp_path) + assert role_blocks == [] + assert grant_blocks == [] + + def test_empty_targets_skipped(self, tmp_path): + data = { + "ROLE": { + "grants": {"DATABASE:USAGE": []}, + } + } + (tmp_path / "tech_role.yaml").write_text(yaml.dump(data)) + _, grant_blocks = generate_tech_roles(tmp_path) + assert grant_blocks == [] + + +# --------------------------------------------------------------------------- +# generate_network_policies +# --------------------------------------------------------------------------- + + +class TestGenerateNetworkPolicies: + """Tests for generate_network_policies.""" + + def test_generates_policy_block(self, tmp_path): + data = { + "office_policy": { + "allowed_ip_list": ["10.0.0.0/8"], + "comment": "Office access", + } + } + (tmp_path / "network_policy.yaml").write_text(yaml.dump(data)) + blocks = generate_network_policies(tmp_path) + assert len(blocks) == 1 + assert "snowflake_network_policy" in blocks[0] + assert "OFFICE_POLICY" in blocks[0] # uppercased + assert "10.0.0.0/8" in blocks[0] + + def test_empty_returns_empty(self, tmp_path): + (tmp_path / "network_policy.yaml").write_text("") + assert generate_network_policies(tmp_path) == [] + + def test_no_file_returns_empty(self, tmp_path): + assert generate_network_policies(tmp_path) == [] + + +# --------------------------------------------------------------------------- +# generate_resource_monitors +# --------------------------------------------------------------------------- + + +class TestGenerateResourceMonitors: + """Tests for generate_resource_monitors.""" + + def test_generates_monitor_block(self, tmp_path): + data = { + "DAILY_MON": { + "credit_quota": 100, + "frequency": "DAILY", + "triggers": { + 80: "NOTIFY", + 95: "SUSPEND", + 100: "SUSPEND_IMMEDIATE", + }, + } + } + (tmp_path / "resource_monitor.yaml").write_text(yaml.dump(data)) + blocks = generate_resource_monitors(tmp_path) + assert len(blocks) == 1 + assert "snowflake_resource_monitor" in blocks[0] + assert "100" in blocks[0] # credit_quota + assert "notify_triggers" in blocks[0] + assert "suspend_triggers" in blocks[0] + assert "suspend_immediate_triggers" in blocks[0] + + def test_no_triggers(self, tmp_path): + data = {"MON": {"credit_quota": 50}} + (tmp_path / "resource_monitor.yaml").write_text(yaml.dump(data)) + blocks = generate_resource_monitors(tmp_path) + assert len(blocks) == 1 + assert "notify_triggers" not in blocks[0] + + def test_empty_returns_empty(self, tmp_path): + (tmp_path / "resource_monitor.yaml").write_text("") + assert generate_resource_monitors(tmp_path) == [] + + +# --------------------------------------------------------------------------- +# generate_databases +# --------------------------------------------------------------------------- + + +class TestGenerateDatabases: + """Tests for generate_databases.""" + + def test_generates_database_block(self, tmp_path): + db_dir = tmp_path / "MY_DB" + db_dir.mkdir() + (db_dir / "params.yaml").write_text(yaml.dump({"comment": "Production DB"})) + blocks = generate_databases(tmp_path) + assert len(blocks) == 1 + assert "snowflake_database" in blocks[0] + assert '"MY_DB"' in blocks[0] + assert "Production DB" in blocks[0] + + def test_sandbox_database(self, tmp_path): + db_dir = tmp_path / "SANDBOX_DB" + db_dir.mkdir() + (db_dir / "params.yaml").write_text(yaml.dump({"is_sandbox": True})) + blocks = generate_databases(tmp_path) + assert len(blocks) == 1 + assert "Sandbox" in blocks[0] or "sandbox" in blocks[0].lower() + + def test_no_databases(self, tmp_path): + blocks = generate_databases(tmp_path) + assert blocks == [] + + def test_multiple_databases_sorted(self, tmp_path): + for name in ["ZOO_DB", "ALPHA_DB"]: + d = tmp_path / name + d.mkdir() + (d / "params.yaml").write_text(yaml.dump({"comment": name})) + blocks = generate_databases(tmp_path) + assert len(blocks) == 2 + assert "ALPHA_DB" in blocks[0] + assert "ZOO_DB" in blocks[1] + + +# --------------------------------------------------------------------------- +# render_section +# --------------------------------------------------------------------------- + + +class TestRenderSection: + """Tests for the render_section helper.""" + + def test_non_empty_blocks(self): + result = render_section("Users", ['resource "snowflake_user" "a" {\n}']) + assert "# Users" in result + assert HEADER in result + assert "snowflake_user" in result + + def test_empty_blocks(self): + """Empty block list returns empty string.""" + result = render_section("Users", []) + assert result == "" + + def test_multiple_blocks_joined(self): + blocks = ["block_a", "block_b"] + result = render_section("Section", blocks) + assert "block_a" in result + assert "block_b" in result + + +# --------------------------------------------------------------------------- +# generate_all (end-to-end) +# --------------------------------------------------------------------------- + + +class TestGenerateAll: + """End-to-end tests for generate_all.""" + + def _write_yaml(self, path, data): + path.write_text(yaml.dump(data, default_flow_style=False)) + + def test_full_generation(self, tmp_path): + """generate_all with a complete config set produces all sections.""" + self._write_yaml( + tmp_path / "user.yaml", + {"ALICE": {"type": "PERSON", "email": "a@b.com"}}, + ) + self._write_yaml( + tmp_path / "warehouse.yaml", + {"WH": {"size": "Small"}}, + ) + self._write_yaml( + tmp_path / "business_role.yaml", + {"ANALYST": {"tech_roles": ["READ_ROLE"]}}, + ) + self._write_yaml( + tmp_path / "tech_role.yaml", + {"READ_ROLE": {"grants": {"DATABASE:USAGE": ["DB"]}}}, + ) + self._write_yaml( + tmp_path / "network_policy.yaml", + {"POL": {"allowed_ip_list": ["10.0.0.0/8"]}}, + ) + self._write_yaml( + tmp_path / "resource_monitor.yaml", + {"MON": {"credit_quota": 50}}, + ) + + output = generate_all(tmp_path) + assert isinstance(output, TerraformOutput) + assert len(output.users) == 1 + assert len(output.warehouses) == 1 + assert len(output.roles) == 2 # 1 business + 1 tech + assert len(output.grants) >= 1 + assert len(output.policies) >= 1 + assert len(output.resource_monitors) == 1 + assert "terraform" in output.main.lower() + + def test_empty_directory(self, tmp_path): + """generate_all with an empty directory returns empty lists.""" + output = generate_all(tmp_path) + assert output.users == [] + assert output.warehouses == [] + assert output.roles == [] + assert output.grants == [] + assert output.resource_monitors == [] + assert output.databases == [] + + +# --------------------------------------------------------------------------- +# write_to_directory +# --------------------------------------------------------------------------- + + +class TestWriteToDirectory: + """Tests for write_to_directory.""" + + def test_creates_files(self, tmp_path): + """write_to_directory creates .tf files in the output directory.""" + output = TerraformOutput() + output.main = "terraform {}" + output.users = ['resource "snowflake_user" "alice" {\n name = "ALICE"\n}'] + output.warehouses = ['resource "snowflake_warehouse" "wh" {\n name = "WH"\n}'] + + out_dir = tmp_path / "terraform_out" + write_to_directory(output, out_dir) + + assert out_dir.is_dir() + assert (out_dir / "main.tf").exists() + assert (out_dir / "users.tf").exists() + assert (out_dir / "warehouses.tf").exists() + + # main.tf should contain the provider block + main_content = (out_dir / "main.tf").read_text() + assert "terraform" in main_content + + def test_skips_empty_sections(self, tmp_path): + """Empty sections do not create files.""" + output = TerraformOutput() + output.main = "terraform {}" + # Everything else empty + + out_dir = tmp_path / "terraform_out" + write_to_directory(output, out_dir) + + assert (out_dir / "main.tf").exists() + assert not (out_dir / "users.tf").exists() + assert not (out_dir / "warehouses.tf").exists() + + def test_creates_parent_directories(self, tmp_path): + """write_to_directory creates parent directories if needed.""" + output = TerraformOutput() + output.main = "terraform {}" + + out_dir = tmp_path / "deep" / "nested" / "dir" + write_to_directory(output, out_dir) + assert out_dir.is_dir() + assert (out_dir / "main.tf").exists() diff --git a/tests/test_validate_config.py b/tests/test_validate_config.py new file mode 100644 index 0000000..5121032 --- /dev/null +++ b/tests/test_validate_config.py @@ -0,0 +1,947 @@ +""" +Test Suite for SnowDDL YAML Configuration Validator. + +Tests the validate_config.py script including: +- ValidationResult dataclass +- YAML loading helpers +- Per-file validators (user, business_role, tech_role, warehouse, network_policy, resource_monitor) +- Cross-reference integrity checks +- End-to-end run_validation orchestrator +""" + +import sys +from pathlib import Path + +# Make scripts/ importable +sys.path.insert(0, str(Path(__file__).parent.parent / "scripts")) + +import pytest +import yaml + +from validate_config import ( + ValidationResult, + _load_yaml, + validate_user_yaml, + validate_business_role_yaml, + validate_tech_role_yaml, + validate_warehouse_yaml, + validate_network_policy_yaml, + validate_resource_monitor_yaml, + run_validation, + _cross_reference_checks, +) + + +# --------------------------------------------------------------------------- +# ValidationResult dataclass +# --------------------------------------------------------------------------- + + +class TestValidationResult: + """Tests for the ValidationResult dataclass.""" + + def test_initial_state(self): + """A fresh result has empty lists and no errors/warnings.""" + result = ValidationResult() + assert result.errors == [] + assert result.warnings == [] + assert result.info == [] + assert result.has_errors is False + assert result.has_warnings is False + + def test_error_method(self): + """error() appends to the errors list.""" + result = ValidationResult() + result.error("something broke") + assert result.errors == ["something broke"] + assert result.has_errors is True + assert result.has_warnings is False + + def test_warning_method(self): + """warning() appends to the warnings list.""" + result = ValidationResult() + result.warning("heads up") + assert result.warnings == ["heads up"] + assert result.has_warnings is True + assert result.has_errors is False + + def test_ok_method(self): + """ok() appends to the info list.""" + result = ValidationResult() + result.ok("all good") + assert result.info == ["all good"] + assert result.has_errors is False + assert result.has_warnings is False + + def test_multiple_messages(self): + """Multiple messages accumulate correctly.""" + result = ValidationResult() + result.error("err1") + result.error("err2") + result.warning("warn1") + result.ok("info1") + assert len(result.errors) == 2 + assert len(result.warnings) == 1 + assert len(result.info) == 1 + + +# --------------------------------------------------------------------------- +# _load_yaml helper +# --------------------------------------------------------------------------- + + +class TestLoadYaml: + """Tests for the _load_yaml helper function.""" + + def test_valid_yaml(self, tmp_path): + """Loading a valid YAML mapping returns (data, None).""" + p = tmp_path / "good.yaml" + p.write_text(yaml.dump({"KEY": {"type": "PERSON"}})) + data, err = _load_yaml(p) + assert err is None + assert data == {"KEY": {"type": "PERSON"}} + + def test_invalid_yaml_syntax(self, tmp_path): + """Loading broken YAML returns (None, error_message).""" + p = tmp_path / "bad.yaml" + p.write_text("key: [unterminated") + data, err = _load_yaml(p) + assert data is None + assert "YAML syntax error" in err + + def test_missing_file(self, tmp_path): + """Loading a non-existent file returns (None, error_message).""" + p = tmp_path / "nonexistent.yaml" + data, err = _load_yaml(p) + assert data is None + assert "File not found" in err + + def test_non_dict_yaml(self, tmp_path): + """Loading a YAML file whose root is a list returns (None, error).""" + p = tmp_path / "list.yaml" + p.write_text("- item1\n- item2\n") + data, err = _load_yaml(p) + assert data is None + assert "Expected a YAML mapping" in err + + def test_empty_yaml(self, tmp_path): + """Loading an empty YAML file returns ({}, None).""" + p = tmp_path / "empty.yaml" + p.write_text("") + data, err = _load_yaml(p) + assert err is None + assert data == {} + + +# --------------------------------------------------------------------------- +# validate_user_yaml +# --------------------------------------------------------------------------- + + +class TestValidateUserYaml: + """Tests for validate_user_yaml.""" + + def test_valid_person_user(self): + """A correctly defined PERSON user produces no errors.""" + data = { + "ALICE": { + "type": "PERSON", + "email": "alice@example.com", + "default_role": "SYSADMIN", + } + } + result = ValidationResult() + validate_user_yaml(data, result, business_roles=set()) + assert not result.has_errors + assert not result.has_warnings + + def test_valid_service_user(self): + """A correctly defined SERVICE user with RSA key produces no errors.""" + data = { + "SVC_PIPELINE": { + "type": "SERVICE", + "rsa_public_key": "MIIBIjANBgkqhkiG9w0BAQE...", + } + } + result = ValidationResult() + validate_user_yaml(data, result, business_roles=set()) + assert not result.has_errors + assert not result.has_warnings + + def test_missing_type_field(self): + """A user without the 'type' field produces an error.""" + data = {"BOB": {"email": "bob@example.com"}} + result = ValidationResult() + validate_user_yaml(data, result, business_roles=set()) + assert result.has_errors + assert any("missing required 'type'" in e for e in result.errors) + + def test_invalid_type_field(self): + """A user with an invalid 'type' value produces an error.""" + data = {"CARL": {"type": "ROBOT"}} + result = ValidationResult() + validate_user_yaml(data, result, business_roles=set()) + assert result.has_errors + assert any("invalid type 'ROBOT'" in e for e in result.errors) + + def test_person_without_email(self): + """A PERSON user without email produces a warning.""" + data = {"DIANE": {"type": "PERSON"}} + result = ValidationResult() + validate_user_yaml(data, result, business_roles=set()) + assert not result.has_errors + assert result.has_warnings + assert any("should have 'email'" in w for w in result.warnings) + + def test_service_without_rsa_key(self): + """A SERVICE user without rsa_public_key produces a warning.""" + data = {"SVC_NO_KEY": {"type": "SERVICE"}} + result = ValidationResult() + validate_user_yaml(data, result, business_roles=set()) + assert result.has_warnings + assert any("should have 'rsa_public_key'" in w for w in result.warnings) + + def test_service_with_placeholder_rsa_key(self): + """A SERVICE user with a placeholder RSA key produces a warning.""" + data = { + "SVC_PLACEHOLDER": { + "type": "SERVICE", + "rsa_public_key": "replace-this-with-real-key", + } + } + result = ValidationResult() + validate_user_yaml(data, result, business_roles=set()) + assert result.has_warnings + assert any("placeholder" in w for w in result.warnings) + + def test_service_with_example_rsa_key(self): + """A SERVICE user with 'example' in RSA key produces a warning.""" + data = { + "SVC_EXAMPLE": { + "type": "SERVICE", + "rsa_public_key": "example-public-key-data", + } + } + result = ValidationResult() + validate_user_yaml(data, result, business_roles=set()) + assert result.has_warnings + assert any("placeholder" in w for w in result.warnings) + + def test_duplicate_users(self): + """Duplicate user names (case-insensitive) produce an error.""" + data = { + "admin_user": {"type": "PERSON", "email": "a@b.com"}, + "ADMIN_USER": {"type": "PERSON", "email": "a@b.com"}, + } + result = ValidationResult() + validate_user_yaml(data, result, business_roles=set()) + assert result.has_errors + assert any("duplicate user name" in e for e in result.errors) + + def test_default_role_cross_reference_valid_system_role(self): + """A default_role pointing at a system role produces no error.""" + data = {"EVE": {"type": "PERSON", "email": "e@b.com", "default_role": "SYSADMIN"}} + result = ValidationResult() + validate_user_yaml(data, result, business_roles={"ANALYST"}) + assert not result.has_errors + + def test_default_role_cross_reference_valid_business_role(self): + """A default_role matching a defined business role produces no error.""" + data = {"FRANK": {"type": "PERSON", "email": "f@b.com", "default_role": "ANALYST"}} + result = ValidationResult() + validate_user_yaml(data, result, business_roles={"ANALYST"}) + assert not result.has_errors + + def test_default_role_cross_reference_with_suffix(self): + """A default_role using __B_ROLE suffix resolves correctly.""" + data = { + "GINA": { + "type": "PERSON", + "email": "g@b.com", + "default_role": "ANALYST__B_ROLE", + } + } + result = ValidationResult() + validate_user_yaml(data, result, business_roles={"ANALYST"}) + assert not result.has_errors + + def test_default_role_cross_reference_invalid(self): + """A default_role not matching any role produces an error.""" + data = { + "HANK": {"type": "PERSON", "email": "h@b.com", "default_role": "NONEXISTENT"} + } + result = ValidationResult() + validate_user_yaml(data, result, business_roles={"ANALYST"}) + assert result.has_errors + assert any("does not match" in e for e in result.errors) + + def test_non_dict_user_entry(self): + """A user whose config is not a dict produces an error.""" + data = {"BAD_USER": "just a string"} + result = ValidationResult() + validate_user_yaml(data, result, business_roles=set()) + assert result.has_errors + assert any("expected a mapping" in e for e in result.errors) + + def test_info_message_count(self): + """After validation, an info message reports the user count.""" + data = { + "U1": {"type": "PERSON", "email": "u1@b.com"}, + "U2": {"type": "SERVICE", "rsa_public_key": "MIIBIj..."}, + } + result = ValidationResult() + validate_user_yaml(data, result, business_roles=set()) + assert any("2 users validated" in i for i in result.info) + + +# --------------------------------------------------------------------------- +# validate_business_role_yaml +# --------------------------------------------------------------------------- + + +class TestValidateBusinessRoleYaml: + """Tests for validate_business_role_yaml.""" + + def test_valid_role(self): + """A valid business role with known tech_roles and warehouses passes.""" + data = { + "ANALYST": { + "tech_roles": ["READ_ROLE"], + "warehouse_usage": ["COMPUTE_WH"], + "schema_owner": ["MY_DB.MY_SCHEMA"], + } + } + result = ValidationResult() + validate_business_role_yaml( + data, result, tech_roles={"READ_ROLE"}, warehouses={"COMPUTE_WH"} + ) + assert not result.has_errors + + def test_missing_tech_role_reference(self): + """Referencing a nonexistent tech_role produces an error.""" + data = {"ANALYST": {"tech_roles": ["GHOST_ROLE"]}} + result = ValidationResult() + validate_business_role_yaml( + data, result, tech_roles={"READ_ROLE"}, warehouses=set() + ) + assert result.has_errors + assert any("GHOST_ROLE" in e for e in result.errors) + + def test_missing_warehouse_reference(self): + """Referencing a nonexistent warehouse produces an error.""" + data = {"ANALYST": {"warehouse_usage": ["GHOST_WH"]}} + result = ValidationResult() + validate_business_role_yaml( + data, result, tech_roles=set(), warehouses={"COMPUTE_WH"} + ) + assert result.has_errors + assert any("GHOST_WH" in e for e in result.errors) + + def test_invalid_schema_owner_format(self): + """A schema_owner not in DB.SCHEMA format produces an error.""" + data = {"ANALYST": {"schema_owner": ["JUST_A_DB"]}} + result = ValidationResult() + validate_business_role_yaml(data, result, tech_roles=set(), warehouses=set()) + assert result.has_errors + assert any("DB.SCHEMA format" in e for e in result.errors) + + def test_valid_schema_owner_format(self): + """A schema_owner in proper DB.SCHEMA format passes.""" + data = {"ANALYST": {"schema_owner": ["PROD_DB.ANALYTICS"]}} + result = ValidationResult() + validate_business_role_yaml(data, result, tech_roles=set(), warehouses=set()) + assert not result.has_errors + + def test_schema_owner_with_special_chars(self): + """A schema_owner with hyphens is invalid (only underscores allowed).""" + data = {"ANALYST": {"schema_owner": ["PROD-DB.ANALYTICS"]}} + result = ValidationResult() + validate_business_role_yaml(data, result, tech_roles=set(), warehouses=set()) + assert result.has_errors + + def test_non_dict_role_entry(self): + """A role whose config is not a dict produces an error.""" + data = {"BAD_ROLE": "not a dict"} + result = ValidationResult() + validate_business_role_yaml(data, result, tech_roles=set(), warehouses=set()) + assert result.has_errors + assert any("expected a mapping" in e for e in result.errors) + + def test_duplicate_role_names(self): + """Duplicate business role names produce an error.""" + data = { + "analyst": {"tech_roles": []}, + "ANALYST": {"tech_roles": []}, + } + result = ValidationResult() + validate_business_role_yaml(data, result, tech_roles=set(), warehouses=set()) + assert result.has_errors + assert any("duplicate role name" in e for e in result.errors) + + def test_empty_tech_roles_no_cross_ref(self): + """When tech_roles set is empty, no cross-reference errors are raised.""" + data = {"ANALYST": {"tech_roles": ["ANY_ROLE"]}} + result = ValidationResult() + validate_business_role_yaml( + data, result, tech_roles=set(), warehouses=set() + ) + assert not result.has_errors + + +# --------------------------------------------------------------------------- +# validate_tech_role_yaml +# --------------------------------------------------------------------------- + + +class TestValidateTechRoleYaml: + """Tests for validate_tech_role_yaml.""" + + def test_valid_grant_keys(self): + """Properly formatted grant keys pass validation.""" + data = { + "READ_ROLE": { + "grants": { + "DATABASE:USAGE": ["MY_DB"], + "SCHEMA:USAGE": ["MY_DB.PUBLIC"], + }, + } + } + result = ValidationResult() + validate_tech_role_yaml(data, result, warehouses=set()) + assert not result.has_errors + + def test_invalid_grant_key_no_colon(self): + """A grant key without a colon produces an error.""" + data = {"BAD_ROLE": {"grants": {"DATABASE_USAGE": ["MY_DB"]}}} + result = ValidationResult() + validate_tech_role_yaml(data, result, warehouses=set()) + assert result.has_errors + assert any("OBJECT_TYPE:PRIVILEGE" in e for e in result.errors) + + def test_invalid_grant_key_empty_privilege(self): + """A grant key with empty privilege after colon produces an error.""" + data = {"BAD_ROLE": {"grants": {"DATABASE:": ["MY_DB"]}}} + result = ValidationResult() + validate_tech_role_yaml(data, result, warehouses=set()) + assert result.has_errors + + def test_invalid_object_type(self): + """An unrecognised object type in grant key produces an error.""" + data = {"BAD_ROLE": {"grants": {"BANANA:USAGE": ["MY_DB"]}}} + result = ValidationResult() + validate_tech_role_yaml(data, result, warehouses=set()) + assert result.has_errors + assert any("invalid object type" in e for e in result.errors) + + def test_warehouse_cross_reference_valid(self): + """WAREHOUSE grants referencing valid warehouses pass.""" + data = { + "WH_ROLE": { + "grants": {"WAREHOUSE:USAGE": ["COMPUTE_WH"]}, + } + } + result = ValidationResult() + validate_tech_role_yaml(data, result, warehouses={"COMPUTE_WH"}) + assert not result.has_errors + + def test_warehouse_cross_reference_invalid(self): + """WAREHOUSE grants referencing unknown warehouses produce an error.""" + data = { + "WH_ROLE": { + "grants": {"WAREHOUSE:USAGE": ["GHOST_WH"]}, + } + } + result = ValidationResult() + validate_tech_role_yaml(data, result, warehouses={"COMPUTE_WH"}) + assert result.has_errors + assert any("GHOST_WH" in e for e in result.errors) + + def test_future_grants_validated_too(self): + """future_grants section is validated the same as grants.""" + data = { + "FG_ROLE": { + "future_grants": {"INVALID_OBJ:SELECT": ["MY_DB"]}, + } + } + result = ValidationResult() + validate_tech_role_yaml(data, result, warehouses=set()) + assert result.has_errors + + def test_non_dict_role_entry(self): + """A tech role whose config is not a dict produces an error.""" + data = {"BAD": 42} + result = ValidationResult() + validate_tech_role_yaml(data, result, warehouses=set()) + assert result.has_errors + + def test_duplicate_tech_role_names(self): + """Duplicate tech role names produce an error.""" + data = { + "read_role": {"grants": {}}, + "READ_ROLE": {"grants": {}}, + } + result = ValidationResult() + validate_tech_role_yaml(data, result, warehouses=set()) + assert result.has_errors + assert any("duplicate role name" in e for e in result.errors) + + +# --------------------------------------------------------------------------- +# validate_warehouse_yaml +# --------------------------------------------------------------------------- + + +class TestValidateWarehouseYaml: + """Tests for validate_warehouse_yaml.""" + + def test_valid_warehouse(self): + """A warehouse with valid size and auto_suspend passes.""" + data = { + "COMPUTE_WH": { + "size": "X-Small", + "auto_suspend": 120, + } + } + result = ValidationResult() + validate_warehouse_yaml(data, result, resource_monitors=set()) + assert not result.has_errors + + def test_valid_warehouse_sizes(self): + """All valid warehouse sizes pass validation.""" + for size in ["X-Small", "Small", "Medium", "Large", "X-Large", "2X-Large"]: + data = {"WH": {"size": size}} + result = ValidationResult() + validate_warehouse_yaml(data, result, resource_monitors=set()) + assert not result.has_errors, f"Size '{size}' should be valid" + + def test_invalid_warehouse_size(self): + """An invalid warehouse size produces an error.""" + data = {"WH": {"size": "Mega"}} + result = ValidationResult() + validate_warehouse_yaml(data, result, resource_monitors=set()) + assert result.has_errors + assert any("invalid size" in e for e in result.errors) + + def test_auto_suspend_valid(self): + """A valid auto_suspend integer passes.""" + data = {"WH": {"auto_suspend": 60}} + result = ValidationResult() + validate_warehouse_yaml(data, result, resource_monitors=set()) + assert not result.has_errors + + def test_auto_suspend_negative(self): + """A negative auto_suspend produces an error.""" + data = {"WH": {"auto_suspend": -1}} + result = ValidationResult() + validate_warehouse_yaml(data, result, resource_monitors=set()) + assert result.has_errors + assert any("positive integer" in e for e in result.errors) + + def test_auto_suspend_non_integer(self): + """A non-integer auto_suspend produces an error.""" + data = {"WH": {"auto_suspend": "fast"}} + result = ValidationResult() + validate_warehouse_yaml(data, result, resource_monitors=set()) + assert result.has_errors + assert any("must be an integer" in e for e in result.errors) + + def test_auto_suspend_zero(self): + """auto_suspend of 0 is valid (means never auto-suspend).""" + data = {"WH": {"auto_suspend": 0}} + result = ValidationResult() + validate_warehouse_yaml(data, result, resource_monitors=set()) + assert not result.has_errors + + def test_resource_monitor_cross_reference_valid(self): + """A valid resource_monitor reference passes.""" + data = {"WH": {"resource_monitor": "MY_MONITOR"}} + result = ValidationResult() + validate_warehouse_yaml(data, result, resource_monitors={"MY_MONITOR"}) + assert not result.has_errors + + def test_resource_monitor_cross_reference_invalid(self): + """An unknown resource_monitor reference produces an error.""" + data = {"WH": {"resource_monitor": "GHOST_MONITOR"}} + result = ValidationResult() + validate_warehouse_yaml(data, result, resource_monitors={"MY_MONITOR"}) + assert result.has_errors + assert any("GHOST_MONITOR" in e for e in result.errors) + + def test_non_dict_warehouse_entry(self): + """A warehouse whose config is not a dict produces an error.""" + data = {"WH": "string-value"} + result = ValidationResult() + validate_warehouse_yaml(data, result, resource_monitors=set()) + assert result.has_errors + + def test_duplicate_warehouse_names(self): + """Duplicate warehouse names produce an error.""" + data = { + "compute_wh": {"size": "Small"}, + "COMPUTE_WH": {"size": "Small"}, + } + result = ValidationResult() + validate_warehouse_yaml(data, result, resource_monitors=set()) + assert result.has_errors + assert any("duplicate warehouse name" in e for e in result.errors) + + +# --------------------------------------------------------------------------- +# validate_network_policy_yaml +# --------------------------------------------------------------------------- + + +class TestValidateNetworkPolicyYaml: + """Tests for validate_network_policy_yaml.""" + + def test_valid_cidr(self): + """Valid CIDR notation in allowed_ip_list passes.""" + data = { + "MY_POLICY": { + "allowed_ip_list": ["10.0.0.0/8", "192.168.1.0/24", "88.216.232.26/32"] + } + } + result = ValidationResult() + validate_network_policy_yaml(data, result) + assert not result.has_errors + + def test_invalid_cidr(self): + """Invalid CIDR notation produces an error.""" + data = {"MY_POLICY": {"allowed_ip_list": ["not-an-ip"]}} + result = ValidationResult() + validate_network_policy_yaml(data, result) + assert result.has_errors + assert any("not valid CIDR" in e for e in result.errors) + + def test_blocked_ip_list_valid(self): + """Valid CIDR in blocked_ip_list passes.""" + data = {"MY_POLICY": {"blocked_ip_list": ["10.0.0.1/32"]}} + result = ValidationResult() + validate_network_policy_yaml(data, result) + assert not result.has_errors + + def test_blocked_ip_list_invalid(self): + """Invalid CIDR in blocked_ip_list produces an error.""" + data = {"MY_POLICY": {"blocked_ip_list": ["garbage"]}} + result = ValidationResult() + validate_network_policy_yaml(data, result) + assert result.has_errors + assert any("blocked IP" in e for e in result.errors) + + def test_non_dict_policy_entry(self): + """A policy whose config is not a dict produces an error.""" + data = {"BAD": 123} + result = ValidationResult() + validate_network_policy_yaml(data, result) + assert result.has_errors + + def test_single_ip_no_cidr_suffix(self): + """A bare IP address (no /mask) is valid CIDR for ip_network(strict=False).""" + data = {"MY_POLICY": {"allowed_ip_list": ["192.168.1.1"]}} + result = ValidationResult() + validate_network_policy_yaml(data, result) + assert not result.has_errors + + def test_duplicate_policy_names(self): + """Duplicate policy names produce an error.""" + data = { + "policy_a": {"allowed_ip_list": ["10.0.0.0/8"]}, + "POLICY_A": {"allowed_ip_list": ["10.0.0.0/8"]}, + } + result = ValidationResult() + validate_network_policy_yaml(data, result) + assert result.has_errors + + +# --------------------------------------------------------------------------- +# validate_resource_monitor_yaml +# --------------------------------------------------------------------------- + + +class TestValidateResourceMonitorYaml: + """Tests for validate_resource_monitor_yaml.""" + + def test_valid_credit_quota(self): + """A positive integer credit_quota passes.""" + data = {"MY_MONITOR": {"credit_quota": 100}} + result = ValidationResult() + validate_resource_monitor_yaml(data, result) + assert not result.has_errors + + def test_invalid_credit_quota_zero(self): + """A credit_quota of 0 produces an error (must be positive).""" + data = {"MY_MONITOR": {"credit_quota": 0}} + result = ValidationResult() + validate_resource_monitor_yaml(data, result) + assert result.has_errors + assert any("must be positive" in e for e in result.errors) + + def test_invalid_credit_quota_negative(self): + """A negative credit_quota produces an error.""" + data = {"MY_MONITOR": {"credit_quota": -50}} + result = ValidationResult() + validate_resource_monitor_yaml(data, result) + assert result.has_errors + + def test_invalid_credit_quota_string(self): + """A non-integer credit_quota produces an error.""" + data = {"MY_MONITOR": {"credit_quota": "lots"}} + result = ValidationResult() + validate_resource_monitor_yaml(data, result) + assert result.has_errors + assert any("must be an integer" in e for e in result.errors) + + def test_no_credit_quota_is_fine(self): + """A monitor without credit_quota is acceptable.""" + data = {"MY_MONITOR": {"frequency": "MONTHLY"}} + result = ValidationResult() + validate_resource_monitor_yaml(data, result) + assert not result.has_errors + + def test_non_dict_monitor_entry(self): + """A monitor whose config is not a dict produces an error.""" + data = {"BAD": "nope"} + result = ValidationResult() + validate_resource_monitor_yaml(data, result) + assert result.has_errors + + +# --------------------------------------------------------------------------- +# _cross_reference_checks +# --------------------------------------------------------------------------- + + +class TestCrossReferenceChecks: + """Tests for _cross_reference_checks.""" + + def _make_loaded(self, **overrides): + """Build a loaded dict with defaults and optional overrides.""" + loaded = { + "user.yaml": {}, + "business_role.yaml": {}, + "tech_role.yaml": {}, + "warehouse.yaml": {}, + "network_policy.yaml": {}, + "resource_monitor.yaml": {}, + } + loaded.update(overrides) + business_roles = {k.upper() for k in loaded["business_role.yaml"]} + tech_roles = {k.upper() for k in loaded["tech_role.yaml"]} + warehouses = {k.upper() for k in loaded["warehouse.yaml"]} + resource_monitors = {k.upper() for k in loaded["resource_monitor.yaml"]} + return loaded, business_roles, tech_roles, warehouses, resource_monitors + + def test_all_valid_cross_refs(self): + """All cross-references resolve -- only ok messages.""" + loaded, br, tr, wh, rm = self._make_loaded( + **{ + "user.yaml": {"ALICE": {"type": "PERSON", "default_role": "ANALYST"}}, + "business_role.yaml": { + "ANALYST": { + "tech_roles": ["READ_ROLE"], + "warehouse_usage": ["COMPUTE_WH"], + } + }, + "tech_role.yaml": { + "READ_ROLE": { + "grants": {"WAREHOUSE:USAGE": ["COMPUTE_WH"]}, + } + }, + "warehouse.yaml": {"COMPUTE_WH": {"size": "Small"}}, + "resource_monitor.yaml": {}, + } + ) + result = ValidationResult() + _cross_reference_checks(loaded, result, br, tr, wh, rm) + assert not result.has_errors + + def test_bad_user_default_role(self): + """User default_role pointing to unknown role is caught.""" + loaded, br, tr, wh, rm = self._make_loaded( + **{ + "user.yaml": {"ALICE": {"type": "PERSON", "default_role": "GHOST"}}, + "business_role.yaml": {"ANALYST": {}}, + } + ) + result = ValidationResult() + _cross_reference_checks(loaded, result, br, tr, wh, rm) + assert result.has_errors + assert any("GHOST" in e for e in result.errors) + + def test_bad_business_role_tech_role_ref(self): + """Business role referencing non-existent tech role is caught.""" + loaded, br, tr, wh, rm = self._make_loaded( + **{ + "business_role.yaml": {"ANALYST": {"tech_roles": ["MISSING"]}}, + } + ) + result = ValidationResult() + _cross_reference_checks(loaded, result, br, tr, wh, rm) + assert result.has_errors + assert any("MISSING" in e for e in result.errors) + + def test_bad_business_role_warehouse_ref(self): + """Business role referencing non-existent warehouse is caught.""" + loaded, br, tr, wh, rm = self._make_loaded( + **{ + "business_role.yaml": {"ANALYST": {"warehouse_usage": ["MISSING_WH"]}}, + } + ) + result = ValidationResult() + _cross_reference_checks(loaded, result, br, tr, wh, rm) + assert result.has_errors + assert any("MISSING_WH" in e for e in result.errors) + + def test_bad_warehouse_resource_monitor_ref(self): + """Warehouse referencing non-existent resource monitor is caught.""" + loaded, br, tr, wh, rm = self._make_loaded( + **{ + "warehouse.yaml": {"WH": {"resource_monitor": "MISSING_MON"}}, + } + ) + result = ValidationResult() + _cross_reference_checks(loaded, result, br, tr, wh, rm) + assert result.has_errors + assert any("MISSING_MON" in e for e in result.errors) + + def test_bad_tech_role_warehouse_grant(self): + """Tech role WAREHOUSE grant referencing unknown warehouse is caught.""" + loaded, br, tr, wh, rm = self._make_loaded( + **{ + "tech_role.yaml": { + "ROLE_X": {"grants": {"WAREHOUSE:USAGE": ["NOPE_WH"]}}, + }, + } + ) + result = ValidationResult() + _cross_reference_checks(loaded, result, br, tr, wh, rm) + assert result.has_errors + assert any("NOPE_WH" in e for e in result.errors) + + def test_system_role_as_default_role(self): + """System roles like SYSADMIN are accepted as user default_role.""" + loaded, br, tr, wh, rm = self._make_loaded( + **{ + "user.yaml": {"ADMIN": {"type": "PERSON", "default_role": "ACCOUNTADMIN"}}, + } + ) + result = ValidationResult() + _cross_reference_checks(loaded, result, br, tr, wh, rm) + assert not result.has_errors + + +# --------------------------------------------------------------------------- +# run_validation (end-to-end) +# --------------------------------------------------------------------------- + + +class TestRunValidation: + """End-to-end tests for run_validation using temporary config directories.""" + + def _write_yaml(self, path, data): + path.write_text(yaml.dump(data, default_flow_style=False)) + + def test_valid_config_returns_zero(self, tmp_path): + """A fully valid config directory returns exit code 0.""" + self._write_yaml( + tmp_path / "user.yaml", + { + "ALICE": { + "type": "PERSON", + "email": "a@b.com", + "default_role": "ANALYST", + } + }, + ) + self._write_yaml( + tmp_path / "business_role.yaml", + { + "ANALYST": { + "tech_roles": ["READ_ROLE"], + "warehouse_usage": ["COMPUTE_WH"], + } + }, + ) + self._write_yaml( + tmp_path / "tech_role.yaml", + {"READ_ROLE": {"grants": {"DATABASE:USAGE": ["MY_DB"]}}}, + ) + self._write_yaml( + tmp_path / "warehouse.yaml", + {"COMPUTE_WH": {"size": "Small", "auto_suspend": 120}}, + ) + self._write_yaml( + tmp_path / "network_policy.yaml", + {"OFFICE_POLICY": {"allowed_ip_list": ["10.0.0.0/8"]}}, + ) + self._write_yaml( + tmp_path / "resource_monitor.yaml", + {"DAILY_MONITOR": {"credit_quota": 50}}, + ) + + exit_code = run_validation(tmp_path, target_files=None, strict=False, quiet=True) + assert exit_code == 0 + + def test_invalid_config_returns_one(self, tmp_path): + """A config with errors returns exit code 1.""" + self._write_yaml( + tmp_path / "user.yaml", + {"BAD_USER": {"email": "no-type@x.com"}}, # missing type + ) + exit_code = run_validation(tmp_path, target_files=None, strict=False, quiet=True) + assert exit_code == 1 + + def test_warnings_only_returns_zero_without_strict(self, tmp_path): + """Warnings without --strict return exit code 0.""" + self._write_yaml( + tmp_path / "user.yaml", + {"SVC": {"type": "SERVICE"}}, # missing rsa key = warning + ) + exit_code = run_validation(tmp_path, target_files=None, strict=False, quiet=True) + assert exit_code == 0 + + def test_warnings_with_strict_returns_one(self, tmp_path): + """Warnings with --strict return exit code 1.""" + self._write_yaml( + tmp_path / "user.yaml", + {"SVC": {"type": "SERVICE"}}, # missing rsa key = warning + ) + exit_code = run_validation(tmp_path, target_files=None, strict=True, quiet=True) + assert exit_code == 1 + + def test_missing_config_dir_returns_one(self, tmp_path): + """A non-existent config directory returns exit code 1.""" + exit_code = run_validation( + tmp_path / "nonexistent", target_files=None, strict=False, quiet=True + ) + assert exit_code == 1 + + def test_target_specific_file(self, tmp_path): + """Validating a specific target file works.""" + self._write_yaml( + tmp_path / "user.yaml", + {"ALICE": {"type": "PERSON", "email": "a@b.com"}}, + ) + self._write_yaml( + tmp_path / "warehouse.yaml", + {"WH": {"size": "Mega"}}, # invalid size = error + ) + # Only validate user.yaml -- should pass despite warehouse errors + exit_code = run_validation( + tmp_path, + target_files=[str(tmp_path / "user.yaml")], + strict=False, + quiet=True, + ) + assert exit_code == 0 + + def test_empty_config_dir_returns_zero(self, tmp_path): + """An empty config directory (no files) returns 0.""" + exit_code = run_validation(tmp_path, target_files=None, strict=False, quiet=True) + assert exit_code == 0 + + def test_yaml_syntax_error_returns_one(self, tmp_path): + """A file with YAML syntax errors returns exit code 1.""" + (tmp_path / "user.yaml").write_text("key: [broken") + exit_code = run_validation(tmp_path, target_files=None, strict=False, quiet=True) + assert exit_code == 1 diff --git a/uv.lock b/uv.lock index b3dacd7..ff0ef91 100644 --- a/uv.lock +++ b/uv.lock @@ -487,6 +487,19 @@ version = "0.9.5" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/f1/2a/8c3ac3d8bc94e6de8d7ae270bb5bc437b210bb9d6d9e46630c98f4abd20c/csscompressor-0.9.5.tar.gz", hash = "sha256:afa22badbcf3120a4f392e4d22f9fff485c044a1feda4a950ecc5eba9dd31a05", size = 237808, upload-time = "2017-11-26T21:13:08.238Z" } +[[package]] +name = "detect-secrets" +version = "1.5.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pyyaml" }, + { name = "requests" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/69/67/382a863fff94eae5a0cf05542179169a1c49a4c8784a9480621e2066ca7d/detect_secrets-1.5.0.tar.gz", hash = "sha256:6bb46dcc553c10df51475641bb30fd69d25645cc12339e46c824c1e0c388898a", size = 97351, upload-time = "2024-05-06T17:46:19.721Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4e/5e/4f5fe4b89fde1dc3ed0eb51bd4ce4c0bca406246673d370ea2ad0c58d747/detect_secrets-1.5.0-py3-none-any.whl", hash = "sha256:e24e7b9b5a35048c313e983f76c4bd09dad89f045ff059e354f9943bf45aa060", size = 120341, upload-time = "2024-05-06T17:46:16.628Z" }, +] + [[package]] name = "distlib" version = "0.4.0" @@ -2347,7 +2360,7 @@ wheels = [ [[package]] name = "snowtower" -version = "0.2.0" +version = "0.3.0" source = { editable = "." } dependencies = [ { name = "click" }, @@ -2399,6 +2412,7 @@ web = [ [package.dev-dependencies] dev = [ + { name = "detect-secrets" }, { name = "faker" }, { name = "hypothesis" }, { name = "mkdocs" }, @@ -2458,6 +2472,7 @@ provides-extras = ["dev", "test", "docs", "web"] [package.metadata.requires-dev] dev = [ + { name = "detect-secrets", specifier = ">=1.5.0" }, { name = "faker", specifier = ">=18.0.0" }, { name = "hypothesis", specifier = ">=6.0.0" }, { name = "mkdocs", specifier = ">=1.6.0" },