From baa7d81f42e4467032b8faf4917762e11f5d60b4 Mon Sep 17 00:00:00 2001 From: database-tycoon Date: Fri, 2 Jan 2026 19:10:53 -0500 Subject: [PATCH 1/4] docs: Add release planning section to maintainer skill MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Added guidance for: - Identifying what goes in a release (issues, roadmap review) - Creating version branches - Pre-release checklist - Release workflow steps - Version naming conventions ๐Ÿค– Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- .claude/skills/snowtower-maintainer/SKILL.md | 56 ++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/.claude/skills/snowtower-maintainer/SKILL.md b/.claude/skills/snowtower-maintainer/SKILL.md index 79a2a5d..0191e20 100644 --- a/.claude/skills/snowtower-maintainer/SKILL.md +++ b/.claude/skills/snowtower-maintainer/SKILL.md @@ -193,6 +193,60 @@ This skill works with: - **Documentation**: `docs/` - **Claude config**: `.claude/` +## Release Planning + +### Identify What Goes in a Release + +1. **Check open issues:** + ```bash + gh issue list --state open --label P1 + gh issue list --state open + ``` + +2. **Review workspace roadmap:** + - Check `/Users/ssciortino/Projects/snowtower-workspace/CLAUDE.md` for: + - "Strategic Recommendations & Next Steps" section + - Priority 1/2/3 items + - Technical debt to address + +3. **Gather completed work since last release:** + ```bash + git log v0.X..main --oneline + ``` + +### Create Version Branch + +```bash +# Create and push version branch +git checkout main +git pull origin main +git checkout -b v0.X +git push -u origin v0.X +``` + +### Pre-Release Checklist + +1. **Run full test suite:** `uv run pytest` +2. **Update CHANGELOG.md** with new features +3. **Verify README is current** (badges, commands, stats) +4. **Check documentation links** resolve correctly +5. **Create PR to main** with release summary + +### Release Workflow + +```bash +# After PR is merged +git checkout main +git pull +gh release create vX.Y --title "vX.Y - Title" --notes "..." +``` + +### Version Naming + +- **v0.X**: Pre-1.0 development releases +- **v1.0**: First stable release (target: installable CLI) +- **vX.Y.Z**: Semantic versioning post-1.0 + ## When to Trigger Invoke this skill when: @@ -203,3 +257,5 @@ Invoke this skill when: - After significant feature additions - Before releases to ensure docs are current - When onboarding new contributors +- **User asks to plan a new release version** +- **User wants to start work on vX.Y** From 1a3fece5da5c404141c56826e9472361cb266895 Mon Sep 17 00:00:00 2001 From: database-tycoon Date: Thu, 22 Jan 2026 09:30:06 -0500 Subject: [PATCH 2/4] feat: Add snowtower-developer Claude Code skill Add comprehensive developer skill for SnowTower contributors with complete development guidance. Features: - Environment setup and architecture overview - UV command creation pattern (4-step process) - Testing strategies (unit, integration, property-based, benchmarks) - Code quality standards (type hints, docstrings, pre-commit) - Debugging workflows and troubleshooting - Pull request process and code review guidelines - Release management procedures - SnowDDL OOP framework usage examples Content: - 817 lines of comprehensive guidance - 30+ practical code examples - 15 major topic sections - 50+ commands documented - 10+ testing patterns Files added: - .claude/skills/snowtower-developer/SKILL.md (main guide) - .claude/skills/snowtower-developer/README.md (quick reference) - .claude/skills/snowtower-developer/CHANGELOG.md (version history) - .claude/skills/README.md (all skills overview) Updated: - README.md (added developer skill to skills table) Closes #TBD Co-Authored-By: Claude Sonnet 4.5 --- .claude/skills/README.md | 218 +++++ .../skills/snowtower-developer/CHANGELOG.md | 105 +++ .claude/skills/snowtower-developer/README.md | 115 +++ .claude/skills/snowtower-developer/SKILL.md | 817 ++++++++++++++++++ README.md | 1 + 5 files changed, 1256 insertions(+) create mode 100644 .claude/skills/README.md create mode 100644 .claude/skills/snowtower-developer/CHANGELOG.md create mode 100644 .claude/skills/snowtower-developer/README.md create mode 100644 .claude/skills/snowtower-developer/SKILL.md diff --git a/.claude/skills/README.md b/.claude/skills/README.md new file mode 100644 index 0000000..42f6de0 --- /dev/null +++ b/.claude/skills/README.md @@ -0,0 +1,218 @@ +# SnowTower Claude Code Skills + +This directory contains Claude Code skills for different user personas working with SnowTower. + +## Available Skills + +### ๐Ÿง‘โ€๐Ÿ’ป [snowtower-developer](./snowtower-developer/) +**For: Code Contributors** + +Comprehensive development guide covering: +- Development environment setup +- Codebase architecture +- Creating UV commands +- Writing tests +- Debugging workflows +- Pull request process +- Code review guidelines + +**Invoke with:** `/snowtower-developer` or mention "developer workflow", "add feature", "write tests" + +--- + +### ๐Ÿ‘จโ€๐Ÿ’ผ [snowtower-admin](./snowtower-admin/) +**For: Infrastructure Administrators** + +Advanced administrative operations including: +- SnowDDL deployments +- User provisioning and lifecycle +- Role and permission management +- Warehouse optimization +- Cost analysis +- Security audits +- Troubleshooting production issues + +**Invoke with:** `/snowtower-admin` or mention "deploy", "snowddl", "user creation", "infrastructure" + +--- + +### ๐Ÿ‘ฅ [snowtower-user](./snowtower-user/) +**For: End Users** + +Getting started and basic operations: +- Requesting Snowflake access +- Generating RSA keys +- Connecting to Snowflake +- Basic SQL queries +- Understanding roles and permissions +- Troubleshooting connection issues + +**Invoke with:** `/snowtower-user` or mention "access request", "connection", "RSA keys" + +--- + +### ๐Ÿ“ [snowtower-maintainer](./snowtower-maintainer/) +**For: Project Maintainers** + +Project maintenance and documentation: +- README updates +- Documentation synchronization +- Claude configuration management +- Project structure audits +- Release management + +**Invoke with:** `/snowtower-maintainer` or mention "documentation", "README", "project maintenance" + +--- + +## Skill Usage Guide + +### How to Invoke a Skill + +In Claude Code, you can invoke skills in several ways: + +1. **Direct command:** + ``` + /snowtower-developer + ``` + +2. **Natural language mention:** + ``` + "Use the developer skill to help me add a new feature" + "I need admin help deploying changes" + "Show me the user guide for connecting to Snowflake" + ``` + +3. **Context-based automatic triggering:** + Skills may auto-trigger based on keywords: + - "deploy", "snowddl" โ†’ admin skill + - "test", "PR", "feature" โ†’ developer skill + - "access", "connection" โ†’ user skill + +### Choosing the Right Skill + +| Task | Skill | Example | +|------|-------|---------| +| Add new feature | Developer | "Help me create a new UV command" | +| Deploy to Snowflake | Admin | "How do I safely deploy these changes?" | +| Request account access | User | "I need to connect to Snowflake" | +| Update documentation | Maintainer | "Sync README with latest changes" | +| Fix a bug | Developer | "Debug this test failure" | +| Create new user | Admin | "Provision a new user account" | +| Setup dev environment | Developer | "Initialize my development environment" | +| Run cost analysis | Admin | "Analyze warehouse costs" | + +### Skill Overlap + +Some tasks may use multiple skills: + +- **Release Management**: Developer (code/tests) + Maintainer (docs/changelog) +- **New User Onboarding**: User (initial access) + Admin (provisioning) +- **Feature Development**: Developer (code) + Admin (infrastructure changes) + +## Contributing to Skills + +Skills are written in Markdown and located in `.claude/skills//SKILL.md`. + +### Adding a New Skill + +1. Create directory: `.claude/skills/my-new-skill/` +2. Create `SKILL.md` with frontmatter: + ```markdown + --- + name: my-new-skill + description: Brief description of when to use this skill + --- + + # Skill content here... + ``` +3. Create `README.md` with overview +4. Update this file to reference the new skill + +### Improving Existing Skills + +1. Edit the `SKILL.md` file +2. Test the skill by invoking it +3. Submit a PR with improvements +4. Document any new patterns discovered + +## Skill Architecture + +Each skill follows this structure: + +``` +snowtower-/ +โ”œโ”€โ”€ SKILL.md # Main skill content (required) +โ”œโ”€โ”€ README.md # Overview and quick reference (recommended) +โ””โ”€โ”€ examples/ # Code examples (optional) +``` + +### SKILL.md Format + +```markdown +--- +name: skill-name +description: Trigger conditions and purpose +--- + +# Skill Title + +## Who This Skill Is For +- Persona 1 +- Persona 2 + +## Quick Command Reference +Common commands + +## Detailed Sections +Topic-specific guidance + +## Examples +Practical examples + +## Troubleshooting +Common issues +``` + +## Best Practices + +1. **Keep skills focused** - Each skill targets a specific persona/use case +2. **Provide examples** - Show, don't just tell +3. **Update regularly** - Keep skills in sync with codebase changes +4. **Cross-reference** - Link to related skills and documentation +5. **Test invocation** - Verify skills can be triggered correctly +6. **Use clear triggers** - Document keywords that activate the skill + +## Integration with Documentation + +Skills complement the main documentation: + +- **Skills**: Interactive, task-oriented, persona-specific +- **Docs**: Comprehensive, reference-oriented, complete coverage + +Users should: +- Use **skills** for guided workflows and common tasks +- Refer to **docs** for deep dives and complete reference + +--- + +## Quick Reference Card + +``` +โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” +โ”‚ SnowTower Skills Quick Reference โ”‚ +โ”œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ค +โ”‚ โ”‚ +โ”‚ ๐Ÿง‘โ€๐Ÿ’ป Developer โ†’ Code, Tests, PRs, Debugging โ”‚ +โ”‚ ๐Ÿ‘จโ€๐Ÿ’ผ Admin โ†’ Deploy, Users, Roles, Warehouses โ”‚ +โ”‚ ๐Ÿ‘ฅ User โ†’ Access, Connect, Basic Usage โ”‚ +โ”‚ ๐Ÿ“ Maintainer โ†’ Docs, README, Project Structure โ”‚ +โ”‚ โ”‚ +โ”‚ Invoke: /snowtower- โ”‚ +โ”‚ โ”‚ +โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ +``` + +--- + +**For more information, see individual skill README files or the main project documentation at `/docs`.** diff --git a/.claude/skills/snowtower-developer/CHANGELOG.md b/.claude/skills/snowtower-developer/CHANGELOG.md new file mode 100644 index 0000000..4e69bad --- /dev/null +++ b/.claude/skills/snowtower-developer/CHANGELOG.md @@ -0,0 +1,105 @@ +# SnowTower Developer Skill - Changelog + +## v1.0.0 - 2026-01-22 + +### Initial Release + +Created comprehensive developer skill for SnowTower contributors with the following sections: + +#### Core Content + +- **Quick Start Guide** - Complete setup instructions for new developers +- **Architecture Overview** - Detailed codebase structure and design patterns +- **Development Workflows** - Step-by-step guides for common tasks +- **Coding Standards** - Python style guide and best practices +- **Testing Guide** - Unit testing, integration testing, mocking strategies +- **Debugging Workflows** - Troubleshooting techniques and tools +- **Pull Request Process** - Complete contribution workflow +- **Advanced Topics** - Dependency injection, CLI extensions, schema validation + +#### Key Features + +1. **UV Command Creation Pattern** + - 4-step process for adding new commands + - Template code examples + - Integration with management_cli.py + - Testing procedures + +2. **Testing Best Practices** + - Mock-based Snowflake testing + - Property-based testing with Hypothesis + - Benchmark testing + - Test organization strategies + +3. **Code Quality Tools** + - Pre-commit hook integration + - Type hints enforcement + - Docstring standards + - Error handling patterns + +4. **SnowDDL OOP Framework Usage** + - Creating objects programmatically + - Reading/writing YAML configurations + - Safety checkpoint system + - Validation framework + +5. **Release Management** + - Pre-release checklist + - Version tagging + - Changelog updates + - CI/CD integration + +#### Documentation + +- Main skill guide: `SKILL.md` (18KB, comprehensive) +- Overview: `README.md` (quick reference) +- Changelog: `CHANGELOG.md` (this file) + +#### Integration + +- Added to main README.md skills table +- Created `.claude/skills/README.md` with all skills overview +- Cross-referenced with existing admin, user, and maintainer skills + +### Purpose + +Enables developers to: +- Set up development environment in <5 minutes +- Understand codebase architecture quickly +- Follow established coding patterns consistently +- Write tests with proper mocking +- Submit high-quality pull requests +- Debug issues efficiently +- Contribute features safely + +### Target Audience + +- New contributors +- Feature developers +- Bug fixers +- QA engineers +- Code reviewers +- Release managers + +### Metrics + +- **Content**: 18KB of comprehensive guidance +- **Sections**: 15 major topics +- **Code Examples**: 30+ practical examples +- **Commands Documented**: 50+ UV commands and workflows +- **Test Patterns**: 10+ testing strategies + +### Future Enhancements + +Planned for future versions: +- Video tutorials integration +- Interactive code examples +- Performance optimization guides +- Advanced CI/CD patterns +- Multi-language support (if applicable) +- Troubleshooting decision trees + +--- + +**Maintained by**: SnowTower Contributors +**Last Updated**: 2026-01-22 diff --git a/.claude/skills/snowtower-developer/README.md b/.claude/skills/snowtower-developer/README.md new file mode 100644 index 0000000..68b5e79 --- /dev/null +++ b/.claude/skills/snowtower-developer/README.md @@ -0,0 +1,115 @@ +# SnowTower Developer Skill + +## Overview + +The `snowtower-developer` skill provides comprehensive guidance for developers contributing to the SnowTower codebase. This skill helps with: + +- Setting up development environment +- Understanding codebase architecture +- Following coding standards and patterns +- Creating new UV commands +- Writing and running tests +- Debugging issues +- Submitting pull requests +- Performing code reviews + +## When to Use This Skill + +Use this skill when you need help with: + +- **Code Contributions**: Adding new features or functionality +- **Bug Fixes**: Debugging and fixing issues +- **Testing**: Writing unit tests, integration tests, or running test suites +- **Development Setup**: Initial environment configuration +- **UV Commands**: Creating new management commands +- **Pull Requests**: Submitting code for review +- **Code Review**: Reviewing pull requests +- **Debugging**: Troubleshooting test failures or code issues + +## How to Invoke + +In Claude Code, you can invoke this skill by mentioning: + +- "Use the developer skill" +- "Help me add a new feature" +- "How do I write tests for this?" +- "Show me how to create a UV command" +- "What's the development workflow?" + +Or explicitly: +``` +/snowtower-developer +``` + +## Skill Contents + +The skill covers: + +1. **Quick Start** - Setting up your development environment +2. **Architecture** - Understanding the codebase structure +3. **Workflows** - Common development tasks +4. **Coding Standards** - Style guide and best practices +5. **Testing** - Writing and running tests +6. **Debugging** - Troubleshooting techniques +7. **Pull Requests** - Contribution workflow +8. **Advanced Topics** - Dependency injection, CLI extensions, validation + +## Related Skills + +- **snowtower-admin**: For infrastructure deployment and operations +- **snowtower-maintainer**: For documentation and project maintenance +- **snowtower-user**: For end-user access and connection help + +## Quick Examples + +### Adding a New Command + +```bash +# 1. Create script +vim scripts/my_feature.py + +# 2. Add wrapper +vim src/management_cli.py + +# 3. Register in pyproject.toml +# 4. Test +uv sync +uv run my-feature --help +``` + +### Running Tests + +```bash +# All tests +uv run pytest -v + +# Specific test +uv run pytest tests/test_user_management.py -v + +# With coverage +uv run pytest --cov=src --cov-report=term +``` + +### Creating a PR + +```bash +git checkout v0.2 +git checkout -b feature/my-feature +# Make changes... +uv run pre-commit run --all-files +uv run pytest -v +git commit -m "feat: Add my feature" +gh pr create --base v0.2 +``` + +## Contributing to This Skill + +If you find ways to improve this skill, please: + +1. Edit `.claude/skills/snowtower-developer/SKILL.md` +2. Submit a PR with your improvements +3. Document any new patterns or workflows you discover + +--- + +**Happy coding! ๐Ÿš€** diff --git a/.claude/skills/snowtower-developer/SKILL.md b/.claude/skills/snowtower-developer/SKILL.md new file mode 100644 index 0000000..1077b1c --- /dev/null +++ b/.claude/skills/snowtower-developer/SKILL.md @@ -0,0 +1,817 @@ +--- +name: snowtower-developer +description: Comprehensive skill for SnowTower contributors and developers. Use when contributing code, fixing bugs, adding features, writing tests, or developing new functionality. Triggers on mentions of development, coding, testing, PR creation, bug fixes, or feature implementation. +--- + +# SnowTower Developer Guide + +A comprehensive skill for developers contributing to the SnowTower codebase. + +## Who This Skill Is For + +- **Contributors** adding new features or fixing bugs +- **Developers** extending SnowTower functionality +- **QA engineers** writing tests +- **Maintainers** reviewing code and managing releases + +--- + +## ๐Ÿš€ Quick Start for New Developers + +### Initial Setup + +```bash +# 1. Fork and clone the repository +git clone https://github.com/YOUR-USERNAME/snowtower-public.git +cd snowtower-public + +# 2. Install UV package manager (if not already installed) +curl -LsSf https://astral.sh/uv/install.sh | sh + +# 3. Create development environment +uv sync --all-extras --dev + +# 4. Set up pre-commit hooks +uv run pre-commit install + +# 5. Create .env file for testing +cp .env.example .env +# Edit .env with test credentials (mock credentials are fine for tests) + +# 6. Verify setup +uv run pytest -v +uv run pre-commit run --all-files +``` + +### First Build + +```bash +# Run all quality checks +uv run pre-commit run --all-files # Linting, formatting, secrets check +uv run pytest -v # Run all 355 tests +uv run pytest --cov=src # With coverage report +``` + +--- + +## ๐Ÿ“ Codebase Architecture + +### Directory Structure + +``` +snowtower-public/ +โ”œโ”€โ”€ src/ # Source code +โ”‚ โ”œโ”€โ”€ snowddl_core/ # OOP framework for SnowDDL +โ”‚ โ”‚ โ”œโ”€โ”€ base.py # Base classes for all objects +โ”‚ โ”‚ โ”œโ”€โ”€ account_objects.py # Users, roles, warehouses +โ”‚ โ”‚ โ”œโ”€โ”€ database_objects.py # Databases, schemas +โ”‚ โ”‚ โ”œโ”€โ”€ validation.py # Validation framework +โ”‚ โ”‚ โ””โ”€โ”€ safety/ # Safety and checkpoint system +โ”‚ โ”œโ”€โ”€ user_management/ # User lifecycle management +โ”‚ โ”‚ โ”œโ”€โ”€ manager.py # UserManager class +โ”‚ โ”‚ โ”œโ”€โ”€ yaml_handler.py # YAML read/write +โ”‚ โ”‚ โ””โ”€โ”€ encryption.py # Fernet encryption +โ”‚ โ”œโ”€โ”€ snowtower_core/ # Core business logic +โ”‚ โ”‚ โ”œโ”€โ”€ models.py # Data models +โ”‚ โ”‚ โ”œโ”€โ”€ metrics.py # Metrics collection +โ”‚ โ”‚ โ””โ”€โ”€ managers.py # Business logic managers +โ”‚ โ”œโ”€โ”€ automation/ # GitHub integration +โ”‚ โ””โ”€โ”€ management_cli.py # CLI entry points +โ”œโ”€โ”€ scripts/ # Management scripts +โ”‚ โ”œโ”€โ”€ manage_users.py # User management +โ”‚ โ”œโ”€โ”€ manage_warehouses.py # Warehouse management +โ”‚ โ”œโ”€โ”€ cost_optimization.py # Cost analysis +โ”‚ โ””โ”€โ”€ deploy_safe.py # Safe deployment wrapper +โ”œโ”€โ”€ tests/ # Test suite +โ”‚ โ”œโ”€โ”€ test_*.py # Unit tests +โ”‚ โ”œโ”€โ”€ integration/ # Integration tests +โ”‚ โ”œโ”€โ”€ manual/ # Manual test scripts +โ”‚ โ””โ”€โ”€ fixtures/ # Test fixtures +โ”œโ”€โ”€ snowddl/ # YAML infrastructure definitions +โ””โ”€โ”€ docs/ # Documentation +``` + +### Key Design Patterns + +1. **Command Pattern**: UV commands wrap scripts in `management_cli.py` +2. **Repository Pattern**: `YAMLHandler` abstracts file operations +3. **Strategy Pattern**: Different authentication methods (RSA vs password) +4. **Factory Pattern**: User creation with type-specific handling +5. **Dependency Injection**: Testable components with mocked dependencies + +--- + +## ๐Ÿ”ง Development Workflows + +### Adding a New UV Command + +**Follow the established 4-step pattern:** + +#### Step 1: Create Script in `scripts/` + +```python +# scripts/my_new_feature.py +from dotenv import load_dotenv +load_dotenv() # MUST BE FIRST LINE + +import argparse +from pathlib import Path +from snowddl_core import SnowflakeClient # Use OOP framework + +def main(): + """Main entry point for my new feature.""" + parser = argparse.ArgumentParser( + description="Description of my feature" + ) + parser.add_argument('--option', help='Example option') + args = parser.parse_args() + + # Implementation here + print(f"Running my feature with option: {args.option}") + +if __name__ == "__main__": + main() +``` + +**Critical Rules:** +- โœ… `load_dotenv()` MUST be the first line +- โœ… Use `argparse` for CLI interface +- โœ… Import from `snowddl_core` OOP framework +- โœ… Include docstrings +- โœ… Add type hints + +#### Step 2: Add Wrapper in `src/management_cli.py` + +```python +def my_new_feature(): + """Run my new feature command.""" + from my_new_feature import main + main() +``` + +#### Step 3: Register in `pyproject.toml` + +```toml +[project.scripts] +my-new-feature = "src.management_cli:my_new_feature" +``` + +#### Step 4: Test the Command + +```bash +# Sync dependencies +uv sync + +# Test command +uv run my-new-feature --help +uv run my-new-feature --option test +``` + +### Writing Tests + +**Test Location Strategy:** +- `tests/test_.py` - Unit tests for `src/.py` +- `tests/integration/` - End-to-end tests +- `tests/manual/` - Complex manual verification scripts + +**Example Unit Test:** + +```python +# tests/test_my_feature.py +"""Tests for my new feature.""" +import pytest +from unittest.mock import Mock, patch +from my_new_feature import main + +class TestMyFeature: + """Test suite for my feature.""" + + def test_basic_functionality(self): + """Test basic feature operation.""" + # Arrange + expected = "expected result" + + # Act + result = my_function() + + # Assert + assert result == expected + + @patch('my_new_feature.SnowflakeClient') + def test_with_mock_snowflake(self, mock_client): + """Test feature with mocked Snowflake connection.""" + # Setup mock + mock_client.return_value.execute.return_value = [] + + # Test with mock + result = main() + + # Verify + mock_client.return_value.execute.assert_called_once() +``` + +**Run Tests:** + +```bash +# Run all tests +uv run pytest -v + +# Run specific test file +uv run pytest tests/test_my_feature.py -v + +# Run with coverage +uv run pytest --cov=src --cov-report=term + +# Run in parallel (faster) +uv run pytest -n auto + +# Run only failed tests from last run +uv run pytest --lf +``` + +--- + +## ๐ŸŽจ Coding Standards + +### Python Style Guide + +**Follow these conventions:** + +1. **Type Hints (Required)** + ```python + def create_user( + username: str, + email: str, + user_type: UserType + ) -> dict[str, Any]: + """Create a new user.""" + pass + ``` + +2. **Docstrings (Required)** + ```python + def process_data(data: list[dict]) -> pd.DataFrame: + """ + Process raw data into structured DataFrame. + + Args: + data: List of dictionaries containing raw data + + Returns: + Pandas DataFrame with processed data + + Raises: + ValueError: If data is empty or malformed + """ + pass + ``` + +3. **Error Handling** + ```python + # Use custom exceptions + from snowddl_core.exceptions import SnowDDLError + + if not username: + raise SnowDDLError("Username is required") + ``` + +4. **Constants** + ```python + # Use uppercase for constants + DEFAULT_WAREHOUSE_SIZE = "X-SMALL" + MAX_PASSWORD_LENGTH = 256 + ``` + +### Pre-commit Checks + +**These run automatically on commit:** + +```yaml +# .pre-commit-config.yaml +- Black (code formatting) +- YAML validation +- Trailing whitespace removal +- Secrets detection (prevent credential leaks) +- File size limits +``` + +**Run manually:** + +```bash +# Check all files +uv run pre-commit run --all-files + +# Check specific files +uv run pre-commit run --files src/my_file.py + +# Skip hooks (rarely needed) +git commit --no-verify +``` + +--- + +## ๐Ÿ› Debugging Workflows + +### Debug a Failing Test + +```bash +# Run with detailed output +uv run pytest tests/test_user_management.py -vv --tb=long + +# Run with debugger +uv run pytest tests/test_user_management.py --pdb + +# Print test output (disable capture) +uv run pytest tests/test_user_management.py -s +``` + +### Debug a Command + +```python +# Add to script for debugging +import pdb; pdb.set_trace() # Breakpoint + +# Or use rich for better output +from rich import print +print(f"Debug info: {variable}") +``` + +### Common Issues + +**Issue: Import errors** +```bash +# Solution: Ensure sync is up to date +uv sync --all-extras --dev +``` + +**Issue: Test failures with Snowflake connection** +```bash +# Solution: Tests use mocks, no real connection needed +# Check your mock setup: +@patch('module.SnowflakeClient') +def test_with_mock(self, mock_client): + # Configure mock to return expected values + mock_client.return_value.execute.return_value = [] +``` + +**Issue: Pre-commit hooks failing** +```bash +# Solution: Auto-fix most issues +uv run pre-commit run --all-files + +# Then commit the fixes +git add . +git commit -m "fix: Apply pre-commit fixes" +``` + +--- + +## ๐Ÿ“ Pull Request Workflow + +### Creating a Feature Branch + +```bash +# Always branch from the release branch (v0.2, v0.3, etc.) +git checkout v0.2 +git pull origin v0.2 +git checkout -b feature/my-awesome-feature +``` + +### Development Cycle + +```bash +# 1. Make changes +vim src/my_file.py + +# 2. Write tests +vim tests/test_my_file.py + +# 3. Run quality checks +uv run pre-commit run --all-files +uv run pytest -v + +# 4. Commit with conventional commit format +git add . +git commit -m "feat: Add my awesome feature" + +# 5. Push and create PR +git push -u origin feature/my-awesome-feature +gh pr create --base v0.2 --title "feat: Add my awesome feature" +``` + +### Commit Message Format + +**Use Conventional Commits:** + +``` +: + +[optional body] + +[optional footer] +``` + +**Types:** +- `feat:` - New feature +- `fix:` - Bug fix +- `docs:` - Documentation changes +- `test:` - Test changes +- `refactor:` - Code refactoring +- `chore:` - Maintenance tasks +- `ci:` - CI/CD changes + +**Examples:** + +```bash +git commit -m "feat: Add user bulk import command" +git commit -m "fix: Correct warehouse auto-suspend calculation" +git commit -m "docs: Update QUICKSTART with new commands" +git commit -m "test: Add integration tests for user management" +``` + +### PR Checklist + +Before submitting, ensure: + +- [ ] All tests pass: `uv run pytest -v` +- [ ] Pre-commit checks pass: `uv run pre-commit run --all-files` +- [ ] New features have tests +- [ ] Documentation is updated +- [ ] Commit messages follow convention +- [ ] No secrets in code +- [ ] Type hints are added +- [ ] Docstrings are complete + +--- + +## ๐Ÿงช Testing Best Practices + +### Test Organization + +```python +# tests/test_user_management.py +class TestUserCreation: + """Tests for user creation functionality.""" + + def setup_method(self): + """Set up test fixtures before each test.""" + self.manager = UserManager(config_directory=Path("/tmp/test")) + + def teardown_method(self): + """Clean up after each test.""" + shutil.rmtree("/tmp/test", ignore_errors=True) + + def test_create_person_user(self): + """Test creating a PERSON type user.""" + result = self.manager.create_user( + first_name="John", + last_name="Doe", + user_type=UserType.PERSON + ) + assert "JOHN_DOE" in result +``` + +### Mocking Snowflake Connections + +```python +# Always mock Snowflake for unit tests +@patch('snowflake.connector.connect') +def test_snowflake_operation(mock_connect): + """Test operation with mocked Snowflake connection.""" + # Setup mock + mock_cursor = Mock() + mock_cursor.fetchall.return_value = [('RESULT',)] + mock_connect.return_value.cursor.return_value = mock_cursor + + # Run test + result = my_snowflake_function() + + # Verify + assert result == [('RESULT',)] + mock_connect.assert_called_once() +``` + +### Property-Based Testing (Advanced) + +```python +from hypothesis import given, strategies as st + +@given( + username=st.text(min_size=1, max_size=50), + email=st.emails() +) +def test_user_creation_with_random_inputs(username, email): + """Test user creation with random inputs (property-based).""" + # This runs 100+ times with different random inputs + result = create_user(username, email) + assert validate_user(result) +``` + +--- + +## ๐Ÿ—๏ธ Using the SnowDDL OOP Framework + +### Creating Objects Programmatically + +```python +from snowddl_core import User, Role, Warehouse, WarehouseSize, UserType +from snowddl_core.safety import create_checkpoint + +# Create a user object +user = User( + name="JOHN_DOE", + comment="Data Analyst", + user_type=UserType.PERSON, + email="john@company.com", + default_role="ANALYST_ROLE" +) + +# Create a warehouse +warehouse = Warehouse( + name="ANALYTICS_WH", + size=WarehouseSize.SMALL, + auto_suspend=300, + auto_resume=True, + comment="Analytics team warehouse" +) + +# Create safety checkpoint before modifications +checkpoint = create_checkpoint("Before user creation") + +# Write to YAML +user.to_yaml(Path("snowddl/user.yaml")) +warehouse.to_yaml(Path("snowddl/warehouse.yaml")) +``` + +### Reading YAML Configurations + +```python +from user_management.yaml_handler import YAMLHandler + +# Read existing configuration +handler = YAMLHandler("snowddl/user.yaml") +users = handler.load() + +# Modify +users["NEW_USER"] = { + "comment": "New data analyst", + "type": "PERSON", + "email": "new@company.com" +} + +# Write back +handler.save(users) +``` + +--- + +## ๐Ÿ“Š Performance Testing + +### Benchmarking Commands + +```bash +# Run performance benchmarks +uv run pytest tests/ --benchmark-only + +# Compare benchmark results +uv run pytest tests/ --benchmark-compare +``` + +### Writing Benchmarks + +```python +def test_user_creation_performance(benchmark): + """Benchmark user creation performance.""" + result = benchmark( + create_user, + username="TEST_USER", + email="test@example.com" + ) + assert result is not None +``` + +--- + +## ๐Ÿ” Code Review Guidelines + +### As a Reviewer + +**Check for:** +- [ ] Tests cover new functionality +- [ ] No hardcoded credentials or secrets +- [ ] Type hints are present +- [ ] Docstrings are complete +- [ ] Error handling is appropriate +- [ ] No breaking changes without migration path +- [ ] CI checks pass +- [ ] Performance implications considered + +### As a Contributor + +**Before requesting review:** +- [ ] Self-review your code +- [ ] Remove debug statements and commented code +- [ ] Ensure variable names are descriptive +- [ ] Add comments for complex logic +- [ ] Update documentation +- [ ] Test edge cases + +--- + +## ๐Ÿšข Release Process + +### Pre-Release Checklist + +See `docs/releases/RELEASE_CHECKLIST.md` for full details. + +**Critical steps:** + +```bash +# 1. Run full test suite +uv run pytest -v + +# 2. All tests MUST pass +# No release with failing tests! + +# 3. Update version +vim pyproject.toml # Update version number + +# 4. Update changelog +vim CHANGELOG.md + +# 5. Create release PR +git checkout -b release/v0.3.0 +git commit -m "chore: Prepare v0.3.0 release" +gh pr create --base main + +# 6. After merge, tag release +git checkout main +git pull +git tag v0.3.0 +git push origin v0.3.0 # Triggers release workflow +``` + +--- + +## ๐ŸŽฏ Common Development Tasks + +### Add a New Snowflake Object Type + +1. **Create class in `src/snowddl_core/`** + ```python + # src/snowddl_core/account_objects.py + class MyNewObject(AccountLevelObject): + """New Snowflake object type.""" + pass + ``` + +2. **Add to exports in `__init__.py`** + +3. **Write tests** + +4. **Update documentation** + +### Add a New Authentication Method + +1. **Extend authentication module** + ```python + # src/user_management/authentication.py + class NewAuthMethod: + """New authentication method.""" + pass + ``` + +2. **Update user YAML schema** + +3. **Add tests with mocks** + +4. **Update security documentation** + +### Add a New Management Command + +See "Adding a New UV Command" section above. + +--- + +## ๐Ÿ“š Essential Documentation + +### For Developers + +- `CLAUDE.md` - Project overview and quick reference +- `docs/llm-context/PATTERNS.md` - Code patterns and conventions +- `docs/contributing/CONTRIBUTING.md` - Contribution guidelines +- `docs/contributing/HOW_TO_TEST.md` - Testing strategies +- `src/snowddl_core/README.md` - OOP framework documentation + +### For Understanding the Domain + +- `docs/guide/ARCHITECTURE.md` - System architecture +- `docs/guide/QUICKSTART.md` - User guide +- `docs/guide/SCHEMA_GRANTS.md` - Schema permission management + +--- + +## ๐Ÿ†˜ Getting Help + +### When Stuck + +1. **Check existing tests** for examples +2. **Read module docstrings** for API documentation +3. **Search GitHub issues** for similar problems +4. **Ask in discussions** or open an issue + +### Useful Commands + +```bash +# See all UV commands +uv run snowtower + +# Get help for specific command +uv run manage-users --help + +# Run interactive Python with project imports +uv run python +>>> from snowddl_core import User +>>> help(User) + +# Check dependencies +uv tree + +# Update dependencies +uv lock --upgrade +``` + +--- + +## โœจ Best Practices Summary + +1. **Always use `load_dotenv()` first** in scripts +2. **Write tests** for all new features +3. **Use type hints** everywhere +4. **Mock Snowflake** connections in tests +5. **Follow conventional commits** format +6. **Run pre-commit** before pushing +7. **Create checkpoints** before destructive operations +8. **Document** public APIs with docstrings +9. **Branch from release branches** not main +10. **Keep PRs focused** on single features + +--- + +## ๐ŸŽ“ Advanced Topics + +### Dependency Injection for Testing + +```python +# Make functions testable by injecting dependencies +def process_users( + config_path: Path, + yaml_handler: Optional[YAMLHandler] = None +): + """Process users with optional injected handler for testing.""" + handler = yaml_handler or YAMLHandler(config_path) + # Use handler... +``` + +### Extending the CLI + +```python +# Add subcommands with click +import click + +@click.group() +def my_command(): + """My command group.""" + pass + +@my_command.command() +@click.option('--verbose', is_flag=True) +def subcommand(verbose): + """A subcommand.""" + if verbose: + print("Verbose mode enabled") +``` + +### Working with YAML Schema Validation + +```python +from pydantic import BaseModel, ValidationError + +class UserSchema(BaseModel): + """Schema for user YAML validation.""" + comment: str + type: str + email: str + +# Validate YAML data +try: + user = UserSchema(**yaml_data) +except ValidationError as e: + print(f"Invalid user config: {e}") +``` + +--- + +**Happy Coding! ๐Ÿš€** + +For questions or improvements to this skill, open an issue or PR on GitHub. diff --git a/README.md b/README.md index 0b119ed..4c13ec5 100644 --- a/README.md +++ b/README.md @@ -629,6 +629,7 @@ This project includes Claude Code skills for AI-assisted workflows: |-------|---------|----------| | [snowtower-user](.claude/skills/snowtower-user/) | End-user guide | Requesting access, connecting to Snowflake | | [snowtower-admin](.claude/skills/snowtower-admin/) | Admin operations | SnowDDL deployments, user management, troubleshooting | +| [snowtower-developer](.claude/skills/snowtower-developer/) | Code contributions | Adding features, fixing bugs, writing tests, submitting PRs | | [snowtower-maintainer](.claude/skills/snowtower-maintainer/) | Project maintenance | README updates, documentation sync | ### Full Documentation From 0b38c7db744556e6c967b26ebb0c77c8685bbfd5 Mon Sep 17 00:00:00 2001 From: database-tycoon Date: Fri, 20 Feb 2026 19:08:32 -0500 Subject: [PATCH 3/4] refactor: Consolidate .claude resources and eliminate duplication Merge unique content from docs/llm-context/ (4 files, ~490 lines) into root CLAUDE.md as single source of truth. Rewrite all 4 skills as concise task checklists (~386 lines total, down from ~1,811). Delete 8 redundant files including skills README, developer CHANGELOG/README, maintainer PROJECT_STRUCTURE, and entire llm-context directory. Net reduction: ~2,840 lines removed across 14 files. Co-Authored-By: Claude Opus 4.6 --- .claude/skills/README.md | 218 ----- .claude/skills/snowtower-admin/SKILL.md | 449 +--------- .../skills/snowtower-developer/CHANGELOG.md | 105 --- .claude/skills/snowtower-developer/README.md | 115 --- .claude/skills/snowtower-developer/SKILL.md | 807 ++---------------- .../snowtower-maintainer/PROJECT_STRUCTURE.md | 71 -- .claude/skills/snowtower-maintainer/SKILL.md | 268 +----- .claude/skills/snowtower-user/SKILL.md | 215 +---- CLAUDE.md | 105 ++- docs/README.md | 13 - docs/llm-context/CLAUDE.md | 329 ------- docs/llm-context/CONTEXT.md | 90 -- docs/llm-context/PATTERNS.md | 491 ----------- docs/llm-context/README.md | 56 -- 14 files changed, 248 insertions(+), 3084 deletions(-) delete mode 100644 .claude/skills/README.md delete mode 100644 .claude/skills/snowtower-developer/CHANGELOG.md delete mode 100644 .claude/skills/snowtower-developer/README.md delete mode 100644 .claude/skills/snowtower-maintainer/PROJECT_STRUCTURE.md delete mode 100644 docs/llm-context/CLAUDE.md delete mode 100644 docs/llm-context/CONTEXT.md delete mode 100644 docs/llm-context/PATTERNS.md delete mode 100644 docs/llm-context/README.md diff --git a/.claude/skills/README.md b/.claude/skills/README.md deleted file mode 100644 index 42f6de0..0000000 --- a/.claude/skills/README.md +++ /dev/null @@ -1,218 +0,0 @@ -# SnowTower Claude Code Skills - -This directory contains Claude Code skills for different user personas working with SnowTower. - -## Available Skills - -### ๐Ÿง‘โ€๐Ÿ’ป [snowtower-developer](./snowtower-developer/) -**For: Code Contributors** - -Comprehensive development guide covering: -- Development environment setup -- Codebase architecture -- Creating UV commands -- Writing tests -- Debugging workflows -- Pull request process -- Code review guidelines - -**Invoke with:** `/snowtower-developer` or mention "developer workflow", "add feature", "write tests" - ---- - -### ๐Ÿ‘จโ€๐Ÿ’ผ [snowtower-admin](./snowtower-admin/) -**For: Infrastructure Administrators** - -Advanced administrative operations including: -- SnowDDL deployments -- User provisioning and lifecycle -- Role and permission management -- Warehouse optimization -- Cost analysis -- Security audits -- Troubleshooting production issues - -**Invoke with:** `/snowtower-admin` or mention "deploy", "snowddl", "user creation", "infrastructure" - ---- - -### ๐Ÿ‘ฅ [snowtower-user](./snowtower-user/) -**For: End Users** - -Getting started and basic operations: -- Requesting Snowflake access -- Generating RSA keys -- Connecting to Snowflake -- Basic SQL queries -- Understanding roles and permissions -- Troubleshooting connection issues - -**Invoke with:** `/snowtower-user` or mention "access request", "connection", "RSA keys" - ---- - -### ๐Ÿ“ [snowtower-maintainer](./snowtower-maintainer/) -**For: Project Maintainers** - -Project maintenance and documentation: -- README updates -- Documentation synchronization -- Claude configuration management -- Project structure audits -- Release management - -**Invoke with:** `/snowtower-maintainer` or mention "documentation", "README", "project maintenance" - ---- - -## Skill Usage Guide - -### How to Invoke a Skill - -In Claude Code, you can invoke skills in several ways: - -1. **Direct command:** - ``` - /snowtower-developer - ``` - -2. **Natural language mention:** - ``` - "Use the developer skill to help me add a new feature" - "I need admin help deploying changes" - "Show me the user guide for connecting to Snowflake" - ``` - -3. **Context-based automatic triggering:** - Skills may auto-trigger based on keywords: - - "deploy", "snowddl" โ†’ admin skill - - "test", "PR", "feature" โ†’ developer skill - - "access", "connection" โ†’ user skill - -### Choosing the Right Skill - -| Task | Skill | Example | -|------|-------|---------| -| Add new feature | Developer | "Help me create a new UV command" | -| Deploy to Snowflake | Admin | "How do I safely deploy these changes?" | -| Request account access | User | "I need to connect to Snowflake" | -| Update documentation | Maintainer | "Sync README with latest changes" | -| Fix a bug | Developer | "Debug this test failure" | -| Create new user | Admin | "Provision a new user account" | -| Setup dev environment | Developer | "Initialize my development environment" | -| Run cost analysis | Admin | "Analyze warehouse costs" | - -### Skill Overlap - -Some tasks may use multiple skills: - -- **Release Management**: Developer (code/tests) + Maintainer (docs/changelog) -- **New User Onboarding**: User (initial access) + Admin (provisioning) -- **Feature Development**: Developer (code) + Admin (infrastructure changes) - -## Contributing to Skills - -Skills are written in Markdown and located in `.claude/skills//SKILL.md`. - -### Adding a New Skill - -1. Create directory: `.claude/skills/my-new-skill/` -2. Create `SKILL.md` with frontmatter: - ```markdown - --- - name: my-new-skill - description: Brief description of when to use this skill - --- - - # Skill content here... - ``` -3. Create `README.md` with overview -4. Update this file to reference the new skill - -### Improving Existing Skills - -1. Edit the `SKILL.md` file -2. Test the skill by invoking it -3. Submit a PR with improvements -4. Document any new patterns discovered - -## Skill Architecture - -Each skill follows this structure: - -``` -snowtower-/ -โ”œโ”€โ”€ SKILL.md # Main skill content (required) -โ”œโ”€โ”€ README.md # Overview and quick reference (recommended) -โ””โ”€โ”€ examples/ # Code examples (optional) -``` - -### SKILL.md Format - -```markdown ---- -name: skill-name -description: Trigger conditions and purpose ---- - -# Skill Title - -## Who This Skill Is For -- Persona 1 -- Persona 2 - -## Quick Command Reference -Common commands - -## Detailed Sections -Topic-specific guidance - -## Examples -Practical examples - -## Troubleshooting -Common issues -``` - -## Best Practices - -1. **Keep skills focused** - Each skill targets a specific persona/use case -2. **Provide examples** - Show, don't just tell -3. **Update regularly** - Keep skills in sync with codebase changes -4. **Cross-reference** - Link to related skills and documentation -5. **Test invocation** - Verify skills can be triggered correctly -6. **Use clear triggers** - Document keywords that activate the skill - -## Integration with Documentation - -Skills complement the main documentation: - -- **Skills**: Interactive, task-oriented, persona-specific -- **Docs**: Comprehensive, reference-oriented, complete coverage - -Users should: -- Use **skills** for guided workflows and common tasks -- Refer to **docs** for deep dives and complete reference - ---- - -## Quick Reference Card - -``` -โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” -โ”‚ SnowTower Skills Quick Reference โ”‚ -โ”œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ค -โ”‚ โ”‚ -โ”‚ ๐Ÿง‘โ€๐Ÿ’ป Developer โ†’ Code, Tests, PRs, Debugging โ”‚ -โ”‚ ๐Ÿ‘จโ€๐Ÿ’ผ Admin โ†’ Deploy, Users, Roles, Warehouses โ”‚ -โ”‚ ๐Ÿ‘ฅ User โ†’ Access, Connect, Basic Usage โ”‚ -โ”‚ ๐Ÿ“ Maintainer โ†’ Docs, README, Project Structure โ”‚ -โ”‚ โ”‚ -โ”‚ Invoke: /snowtower- โ”‚ -โ”‚ โ”‚ -โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ -``` - ---- - -**For more information, see individual skill README files or the main project documentation at `/docs`.** diff --git a/.claude/skills/snowtower-admin/SKILL.md b/.claude/skills/snowtower-admin/SKILL.md index ad4ebe8..84864fb 100644 --- a/.claude/skills/snowtower-admin/SKILL.md +++ b/.claude/skills/snowtower-admin/SKILL.md @@ -5,162 +5,50 @@ description: Advanced skill for SnowTower infrastructure administrators. Use for # SnowTower Administrator Guide -A comprehensive skill for administrators managing Snowflake infrastructure through SnowTower. +Assumes CLAUDE.md is loaded for project context, role hierarchy, and SnowDDL knowledge. -## Who This Skill Is For - -- **Infrastructure administrators** managing SnowDDL deployments -- **Security admins** handling user provisioning and roles -- **DevOps engineers** managing CI/CD pipelines -- **On-call engineers** troubleshooting production issues - ---- - -## Quick Command Reference +## Deployment Checklist ```bash -# Essential commands -uv run snowddl-plan # Preview changes (ALWAYS run first) -uv run deploy-safe # Apply changes safely -uv run manage-users # User lifecycle management -uv run manage-warehouses # Warehouse operations -uv run manage-costs # Cost analysis -``` - ---- - -## Core Operations - -### SnowDDL Deployment Workflow - -**CRITICAL: Always use `deploy-safe`, never raw `snowddl-apply`** - -```bash -# 1. Make changes to YAML files in snowddl/ -vim snowddl/user.yaml - +# 1. Edit YAML files in snowddl/ # 2. ALWAYS preview first uv run snowddl-plan - -# 3. Review the plan output carefully -# Look for: CREATE, ALTER, DROP, GRANT, REVOKE statements - +# 3. Review output for unexpected DROP/REVOKE statements # 4. Apply using safe deployment (preserves schema grants) uv run deploy-safe ``` -**Why `deploy-safe`?** -SnowDDL excludes SCHEMA objects from management, which can cause it to revoke schema-level grants. The `deploy-safe` wrapper automatically restores these grants after every deployment, preventing dbt and other tools from losing permissions. - -### Understanding Plan Output - -``` -[APPLY] CREATE USER "NEW_USER" โ† New object will be created -[APPLY] ALTER USER "EXISTING_USER" โ† Object will be modified -[APPLY] DROP USER "OLD_USER" โ† Object will be deleted (CAREFUL!) -[APPLY] GRANT ROLE "X" TO USER "Y" โ† Permission will be added -[APPLY] REVOKE ROLE "X" FROM USER "Y" โ† Permission will be removed -``` - -**Red flags to watch for:** -- Unexpected `DROP` statements -- Mass `REVOKE` statements (might be schema drift) -- Changes to admin roles (ACCOUNTADMIN, SECURITYADMIN) - ---- +**Why `deploy-safe`?** Raw `snowddl-apply` can revoke schema-level grants managed by dbt. The wrapper restores them automatically. -## User Management +**Plan output flags:** +- `CREATE` / `ALTER` - normally expected +- `DROP` - verify this is intentional +- Mass `REVOKE` - likely schema drift (see CLAUDE.md "Schema Drift Problem") +- Changes to ACCOUNTADMIN/SECURITYADMIN - red flag -### Creating a New User +## User Creation -**Option 1: Interactive wizard (recommended)** +**Option 1: Interactive wizard** (recommended) ```bash uv run manage-users create ``` -**Option 2: Edit YAML directly** -```yaml -# snowddl/user.yaml -NEW_USER: - comment: "Data Analyst - Analytics Team" - type: PERSON - default_role: ANALYST_ROLE__B_ROLE - default_warehouse: MAIN_WAREHOUSE - email: user@company.com - authentication: - password: !decrypt | - gAAAAABl...encrypted... - rsa_public_key: | - -----BEGIN PUBLIC KEY----- - MIIBIjAN... - -----END PUBLIC KEY----- -``` +**Option 2: Edit YAML directly** in `snowddl/user.yaml` **Option 3: Non-interactive** ```bash -uv run manage-users create \ - --first-name Jane \ - --last-name Smith \ - --email jane@company.com \ - --role ANALYST_ROLE +uv run manage-users create --first-name Jane --last-name Smith --email jane@company.com --role ANALYST_ROLE ``` -### User Types - -| Type | Use For | MFA Required | Network Policy | -|------|---------|--------------|----------------| -| `PERSON` | Human users | Yes (by 2026) | Applied | -| `SERVICE` | Service accounts | No | Not applied | - -### Encrypting Passwords - +**Encrypt passwords:** ```bash -# Generate Fernet key (one-time setup) -uv run util-generate-key - -# Encrypt a password uv run snowddl-encrypt "MySecurePassword123!" -# Output: gAAAAABl... - -# Use in YAML with !decrypt tag -authentication: - password: !decrypt | - gAAAAABl...encrypted_output... -``` - -### RSA Key Setup for Users - -```bash -# Generate keys for a user -uv run generate-rsa-batch --users NEW_USER - -# Or manually: -openssl genrsa 2048 | openssl pkcs8 -topk8 -inform PEM -nocrypt -out user_key.p8 -openssl rsa -in user_key.p8 -pubout -out user_key.pub - -# Add public key to user.yaml -cat user_key.pub -``` - ---- - -## Role Hierarchy - -### SnowDDL Role Naming Convention - -``` -ROLE_NAME__B_ROLE โ†’ Business role (assigned to users) -ROLE_NAME__T_ROLE โ†’ Technical role (assigned to business roles) -DB__SCHEMA__S_ROLE โ†’ Schema role (auto-created by SnowDDL) +# Use output in YAML with !decrypt tag ``` -### Role Assignment Flow +**For service accounts:** See [Service Account Pattern](../../.claude/patterns/SERVICE_ACCOUNT_CREATION_PATTERN.md) -``` -User โ†’ Business Role (__B_ROLE) โ†’ Technical Roles (__T_ROLE) โ†’ Permissions -``` - -### Creating Roles +## Role Management **Business Role** (`snowddl/business_role.yaml`): ```yaml @@ -168,74 +56,45 @@ ANALYST_ROLE: comment: "Business analysts with read access" tech_roles: - STRIPE_READER_ROLE - - ANALYTICS_READER_ROLE warehouse_usage: - MAIN_WAREHOUSE schema_read: - PROJ_STRIPE.ANALYTICS ``` -**Technical Role** (`snowddl/tech_role.yaml`): -```yaml -STRIPE_READER_ROLE: - grants: - DATABASE:USAGE: - - SOURCE_STRIPE - - PROJ_STRIPE - SCHEMA:USAGE: - - SOURCE_STRIPE.STRIPE_WHY - - PROJ_STRIPE.PROJ_STRIPE - TABLE:SELECT: - - SOURCE_STRIPE.STRIPE_WHY.* -``` +**Technical Role** (`snowddl/tech_role.yaml`): See grant format in CLAUDE.md "SnowDDL Object Type Reference". ---- +## Troubleshooting Quick Fixes -## Database & Schema Management +| Problem | Fix | +|---------|-----| +| Schema grant drift (mass REVOKEs) | Use `uv run deploy-safe` instead of raw apply | +| "Object does not exist" | Ensure using `-r ACCOUNTADMIN` | +| "Insufficient privileges" | Check `SNOWFLAKE_ROLE=ACCOUNTADMIN` in `.env` | +| Exit code 8 | Not an error - means changes were applied | +| User locked out | `ALTER USER USERNAME SET MINS_TO_UNLOCK = 0;` | -### Creating a Database +## Emergency Procedures +**Rollback last deployment:** ```bash -# Create directory -mkdir snowddl/MY_NEW_DB - -# Add params.yaml -cat > snowddl/MY_NEW_DB/params.yaml << 'EOF' -comment: "New database for analytics project" -is_transient: false -EOF - -# Deploy -uv run snowddl-plan +git checkout HEAD~1 -- snowddl/ uv run deploy-safe ``` -### Creating a Schema - +**Service account reset:** ```bash -# Create schema directory -mkdir snowddl/MY_DB/MY_SCHEMA - -# Add params.yaml -cat > snowddl/MY_DB/MY_SCHEMA/params.yaml << 'EOF' -comment: "Schema for raw data ingestion" -is_transient: false -is_sandbox: false -EOF +uv run generate-rsa-batch --users SNOWDDL --force +snow sql -q "ALTER USER SNOWDDL SET RSA_PUBLIC_KEY='...'" +gh secret set SNOWFLAKE_PRIVATE_KEY < new_key.p8 ``` -### Schema Types - -| Parameter | Effect | -|-----------|--------| -| `is_transient: true` | No Time Travel, no Fail-safe | -| `is_sandbox: true` | Creates as TRANSIENT schema | - ---- - -## CI/CD Operations +**MFA compliance check:** +```bash +uv run manage-security --check-mfa +``` -### GitHub Actions Workflows +## CI/CD Workflows | Workflow | Trigger | Purpose | |----------|---------|---------| @@ -243,233 +102,3 @@ EOF | `release.yml` | Tags `v*` | Create GitHub release | | `labeler.yml` | PRs | Auto-label by file type | | `changelog.yml` | Push to main | Update changelog | - -### Making Infrastructure Changes via PR - -```bash -# 1. Create feature branch -git checkout v0.2 -git checkout -b feature/add-new-user - -# 2. Make YAML changes -vim snowddl/user.yaml - -# 3. Validate locally -uv run pre-commit run --all-files -uv run pytest - -# 4. Commit and push -git add . -git commit -m "feat: Add new user JANE_DOE" -git push -u origin feature/add-new-user - -# 5. Create PR -gh pr create --base v0.2 - -# 6. CI runs automatically, merge after approval -``` - -### Release Process - -```bash -# After all PRs merged to v0.2 -git checkout main -git pull -git merge v0.2 -git tag v0.2.0 -git push origin main --tags -# Release workflow creates GitHub release automatically -``` - ---- - -## Troubleshooting - -### Schema Grant Drift - -**Symptom:** Plan shows hundreds of REVOKE statements for schema grants - -**Cause:** SnowDDL doesn't manage SCHEMA objects directly; grants from dbt or other tools appear as drift - -**Solution:** -```bash -# Always use deploy-safe which auto-applies schema grants -uv run deploy-safe - -# Or manually apply schema grants -uv run apply-schema-grants -``` - -### Authentication Failures - -```bash -# Diagnose authentication issues -uv run util-diagnose-auth - -# Fix common auth problems -uv run util-fix-auth - -# Check specific user -snow sql -q "DESCRIBE USER USERNAME" -``` - -### User Locked Out - -```sql --- Check user status -SHOW USERS LIKE 'USERNAME'; - --- Unlock user (as ACCOUNTADMIN) -ALTER USER USERNAME SET MINS_TO_UNLOCK = 0; - --- Check login history -SELECT * FROM SNOWFLAKE.ACCOUNT_USAGE.LOGIN_HISTORY -WHERE USER_NAME = 'USERNAME' -ORDER BY EVENT_TIMESTAMP DESC -LIMIT 10; -``` - -### SnowDDL Errors - -**"Object does not exist"** -- Run with ACCOUNTADMIN: `uv run snowddl-plan` uses `-r ACCOUNTADMIN` -- Check object was created in correct database/schema - -**"Insufficient privileges"** -- Verify SNOWFLAKE_ROLE is set to ACCOUNTADMIN -- Check the service account has required permissions - -**Exit code 8** -- Means "changes applied successfully" -- Not an error, SnowDDL uses this to indicate modifications were made - ---- - -## Security Operations - -### Network Policies - -```yaml -# snowddl/network_policy.yaml -corporate_network_policy: - allowed_ip_list: - - 192.0.2.0/24 - - 10.0.0.0/8 - comment: "Corporate network access only" -``` - -### MFA Compliance - -**Deadline:** March 2026 for all human users - -```bash -# Check MFA status -uv run manage-security --check-mfa - -# List users without MFA -snow sql -q " - SELECT name, has_mfa_registered - FROM SNOWFLAKE.ACCOUNT_USAGE.USERS - WHERE type = 'PERSON' AND NOT has_mfa_registered -" -``` - -### Emergency Access - -The `STEPHEN_RECOVERY` account is configured without network policy for emergency access: -- Use only when primary access methods fail -- Requires password authentication -- Document all usage - ---- - -## Cost Management - -```bash -# Analyze costs -uv run manage-costs --analyze - -# Check warehouse usage -uv run manage-warehouses --status - -# Suspend all warehouses (emergency) -uv run manage-warehouses --suspend-all -``` - -### Warehouse Configuration - -```yaml -# snowddl/warehouse.yaml -MAIN_WAREHOUSE: - size: XSMALL - auto_suspend: 60 # seconds - auto_resume: true - min_cluster_count: 1 - max_cluster_count: 1 - resource_monitor: main_monitor -``` - ---- - -## Key File Locations - -| File | Purpose | -|------|---------| -| `snowddl/user.yaml` | User accounts | -| `snowddl/business_role.yaml` | Business roles | -| `snowddl/tech_role.yaml` | Technical roles with grants | -| `snowddl/warehouse.yaml` | Warehouse configuration | -| `snowddl/*_policy.yaml` | Security policies | -| `snowddl/{DB}/params.yaml` | Database configuration | -| `snowddl/{DB}/{SCHEMA}/params.yaml` | Schema configuration | - ---- - -## Environment Variables - -Required in `.env`: -```bash -SNOWFLAKE_ACCOUNT=your_account -SNOWFLAKE_USER=SNOWDDL -SNOWFLAKE_ROLE=ACCOUNTADMIN -SNOWFLAKE_WAREHOUSE=ADMIN -SNOWFLAKE_PRIVATE_KEY_PATH=~/.ssh/snowflake_rsa_key.p8 -SNOWFLAKE_CONFIG_FERNET_KEYS=your_fernet_key -``` - ---- - -## Emergency Procedures - -### Rollback Last Deployment - -```bash -# Revert YAML changes -git checkout HEAD~1 -- snowddl/ - -# Re-apply -uv run deploy-safe -``` - -### Complete Service Account Reset - -```bash -# Regenerate RSA keys -uv run generate-rsa-batch --users SNOWDDL --force - -# Update in Snowflake -snow sql -q "ALTER USER SNOWDDL SET RSA_PUBLIC_KEY='...'" - -# Update GitHub secrets -gh secret set SNOWFLAKE_PRIVATE_KEY < new_key.p8 -``` - -### Health Check - -```bash -# Quick health check -uv run monitor-health - -# Full system audit -uv run manage-security --full-audit -``` diff --git a/.claude/skills/snowtower-developer/CHANGELOG.md b/.claude/skills/snowtower-developer/CHANGELOG.md deleted file mode 100644 index 4e69bad..0000000 --- a/.claude/skills/snowtower-developer/CHANGELOG.md +++ /dev/null @@ -1,105 +0,0 @@ -# SnowTower Developer Skill - Changelog - -## v1.0.0 - 2026-01-22 - -### Initial Release - -Created comprehensive developer skill for SnowTower contributors with the following sections: - -#### Core Content - -- **Quick Start Guide** - Complete setup instructions for new developers -- **Architecture Overview** - Detailed codebase structure and design patterns -- **Development Workflows** - Step-by-step guides for common tasks -- **Coding Standards** - Python style guide and best practices -- **Testing Guide** - Unit testing, integration testing, mocking strategies -- **Debugging Workflows** - Troubleshooting techniques and tools -- **Pull Request Process** - Complete contribution workflow -- **Advanced Topics** - Dependency injection, CLI extensions, schema validation - -#### Key Features - -1. **UV Command Creation Pattern** - - 4-step process for adding new commands - - Template code examples - - Integration with management_cli.py - - Testing procedures - -2. **Testing Best Practices** - - Mock-based Snowflake testing - - Property-based testing with Hypothesis - - Benchmark testing - - Test organization strategies - -3. **Code Quality Tools** - - Pre-commit hook integration - - Type hints enforcement - - Docstring standards - - Error handling patterns - -4. **SnowDDL OOP Framework Usage** - - Creating objects programmatically - - Reading/writing YAML configurations - - Safety checkpoint system - - Validation framework - -5. **Release Management** - - Pre-release checklist - - Version tagging - - Changelog updates - - CI/CD integration - -#### Documentation - -- Main skill guide: `SKILL.md` (18KB, comprehensive) -- Overview: `README.md` (quick reference) -- Changelog: `CHANGELOG.md` (this file) - -#### Integration - -- Added to main README.md skills table -- Created `.claude/skills/README.md` with all skills overview -- Cross-referenced with existing admin, user, and maintainer skills - -### Purpose - -Enables developers to: -- Set up development environment in <5 minutes -- Understand codebase architecture quickly -- Follow established coding patterns consistently -- Write tests with proper mocking -- Submit high-quality pull requests -- Debug issues efficiently -- Contribute features safely - -### Target Audience - -- New contributors -- Feature developers -- Bug fixers -- QA engineers -- Code reviewers -- Release managers - -### Metrics - -- **Content**: 18KB of comprehensive guidance -- **Sections**: 15 major topics -- **Code Examples**: 30+ practical examples -- **Commands Documented**: 50+ UV commands and workflows -- **Test Patterns**: 10+ testing strategies - -### Future Enhancements - -Planned for future versions: -- Video tutorials integration -- Interactive code examples -- Performance optimization guides -- Advanced CI/CD patterns -- Multi-language support (if applicable) -- Troubleshooting decision trees - ---- - -**Maintained by**: SnowTower Contributors -**Last Updated**: 2026-01-22 diff --git a/.claude/skills/snowtower-developer/README.md b/.claude/skills/snowtower-developer/README.md deleted file mode 100644 index 68b5e79..0000000 --- a/.claude/skills/snowtower-developer/README.md +++ /dev/null @@ -1,115 +0,0 @@ -# SnowTower Developer Skill - -## Overview - -The `snowtower-developer` skill provides comprehensive guidance for developers contributing to the SnowTower codebase. This skill helps with: - -- Setting up development environment -- Understanding codebase architecture -- Following coding standards and patterns -- Creating new UV commands -- Writing and running tests -- Debugging issues -- Submitting pull requests -- Performing code reviews - -## When to Use This Skill - -Use this skill when you need help with: - -- **Code Contributions**: Adding new features or functionality -- **Bug Fixes**: Debugging and fixing issues -- **Testing**: Writing unit tests, integration tests, or running test suites -- **Development Setup**: Initial environment configuration -- **UV Commands**: Creating new management commands -- **Pull Requests**: Submitting code for review -- **Code Review**: Reviewing pull requests -- **Debugging**: Troubleshooting test failures or code issues - -## How to Invoke - -In Claude Code, you can invoke this skill by mentioning: - -- "Use the developer skill" -- "Help me add a new feature" -- "How do I write tests for this?" -- "Show me how to create a UV command" -- "What's the development workflow?" - -Or explicitly: -``` -/snowtower-developer -``` - -## Skill Contents - -The skill covers: - -1. **Quick Start** - Setting up your development environment -2. **Architecture** - Understanding the codebase structure -3. **Workflows** - Common development tasks -4. **Coding Standards** - Style guide and best practices -5. **Testing** - Writing and running tests -6. **Debugging** - Troubleshooting techniques -7. **Pull Requests** - Contribution workflow -8. **Advanced Topics** - Dependency injection, CLI extensions, validation - -## Related Skills - -- **snowtower-admin**: For infrastructure deployment and operations -- **snowtower-maintainer**: For documentation and project maintenance -- **snowtower-user**: For end-user access and connection help - -## Quick Examples - -### Adding a New Command - -```bash -# 1. Create script -vim scripts/my_feature.py - -# 2. Add wrapper -vim src/management_cli.py - -# 3. Register in pyproject.toml -# 4. Test -uv sync -uv run my-feature --help -``` - -### Running Tests - -```bash -# All tests -uv run pytest -v - -# Specific test -uv run pytest tests/test_user_management.py -v - -# With coverage -uv run pytest --cov=src --cov-report=term -``` - -### Creating a PR - -```bash -git checkout v0.2 -git checkout -b feature/my-feature -# Make changes... -uv run pre-commit run --all-files -uv run pytest -v -git commit -m "feat: Add my feature" -gh pr create --base v0.2 -``` - -## Contributing to This Skill - -If you find ways to improve this skill, please: - -1. Edit `.claude/skills/snowtower-developer/SKILL.md` -2. Submit a PR with your improvements -3. Document any new patterns or workflows you discover - ---- - -**Happy coding! ๐Ÿš€** diff --git a/.claude/skills/snowtower-developer/SKILL.md b/.claude/skills/snowtower-developer/SKILL.md index 1077b1c..875e868 100644 --- a/.claude/skills/snowtower-developer/SKILL.md +++ b/.claude/skills/snowtower-developer/SKILL.md @@ -5,813 +5,112 @@ description: Comprehensive skill for SnowTower contributors and developers. Use # SnowTower Developer Guide -A comprehensive skill for developers contributing to the SnowTower codebase. +Assumes CLAUDE.md is loaded for project context and command patterns. -## Who This Skill Is For - -- **Contributors** adding new features or fixing bugs -- **Developers** extending SnowTower functionality -- **QA engineers** writing tests -- **Maintainers** reviewing code and managing releases - ---- - -## ๐Ÿš€ Quick Start for New Developers - -### Initial Setup +## Setup ```bash -# 1. Fork and clone the repository +# 1. Fork and clone git clone https://github.com/YOUR-USERNAME/snowtower-public.git cd snowtower-public -# 2. Install UV package manager (if not already installed) +# 2. Install UV (if needed) curl -LsSf https://astral.sh/uv/install.sh | sh -# 3. Create development environment +# 3. Install dependencies uv sync --all-extras --dev # 4. Set up pre-commit hooks uv run pre-commit install -# 5. Create .env file for testing +# 5. Create .env (mock credentials fine for tests) cp .env.example .env -# Edit .env with test credentials (mock credentials are fine for tests) -# 6. Verify setup +# 6. Verify uv run pytest -v uv run pre-commit run --all-files ``` -### First Build - -```bash -# Run all quality checks -uv run pre-commit run --all-files # Linting, formatting, secrets check -uv run pytest -v # Run all 355 tests -uv run pytest --cov=src # With coverage report -``` - ---- - -## ๐Ÿ“ Codebase Architecture - -### Directory Structure - -``` -snowtower-public/ -โ”œโ”€โ”€ src/ # Source code -โ”‚ โ”œโ”€โ”€ snowddl_core/ # OOP framework for SnowDDL -โ”‚ โ”‚ โ”œโ”€โ”€ base.py # Base classes for all objects -โ”‚ โ”‚ โ”œโ”€โ”€ account_objects.py # Users, roles, warehouses -โ”‚ โ”‚ โ”œโ”€โ”€ database_objects.py # Databases, schemas -โ”‚ โ”‚ โ”œโ”€โ”€ validation.py # Validation framework -โ”‚ โ”‚ โ””โ”€โ”€ safety/ # Safety and checkpoint system -โ”‚ โ”œโ”€โ”€ user_management/ # User lifecycle management -โ”‚ โ”‚ โ”œโ”€โ”€ manager.py # UserManager class -โ”‚ โ”‚ โ”œโ”€โ”€ yaml_handler.py # YAML read/write -โ”‚ โ”‚ โ””โ”€โ”€ encryption.py # Fernet encryption -โ”‚ โ”œโ”€โ”€ snowtower_core/ # Core business logic -โ”‚ โ”‚ โ”œโ”€โ”€ models.py # Data models -โ”‚ โ”‚ โ”œโ”€โ”€ metrics.py # Metrics collection -โ”‚ โ”‚ โ””โ”€โ”€ managers.py # Business logic managers -โ”‚ โ”œโ”€โ”€ automation/ # GitHub integration -โ”‚ โ””โ”€โ”€ management_cli.py # CLI entry points -โ”œโ”€โ”€ scripts/ # Management scripts -โ”‚ โ”œโ”€โ”€ manage_users.py # User management -โ”‚ โ”œโ”€โ”€ manage_warehouses.py # Warehouse management -โ”‚ โ”œโ”€โ”€ cost_optimization.py # Cost analysis -โ”‚ โ””โ”€โ”€ deploy_safe.py # Safe deployment wrapper -โ”œโ”€โ”€ tests/ # Test suite -โ”‚ โ”œโ”€โ”€ test_*.py # Unit tests -โ”‚ โ”œโ”€โ”€ integration/ # Integration tests -โ”‚ โ”œโ”€โ”€ manual/ # Manual test scripts -โ”‚ โ””โ”€โ”€ fixtures/ # Test fixtures -โ”œโ”€โ”€ snowddl/ # YAML infrastructure definitions -โ””โ”€โ”€ docs/ # Documentation -``` - -### Key Design Patterns - -1. **Command Pattern**: UV commands wrap scripts in `management_cli.py` -2. **Repository Pattern**: `YAMLHandler` abstracts file operations -3. **Strategy Pattern**: Different authentication methods (RSA vs password) -4. **Factory Pattern**: User creation with type-specific handling -5. **Dependency Injection**: Testable components with mocked dependencies - ---- - -## ๐Ÿ”ง Development Workflows - -### Adding a New UV Command - -**Follow the established 4-step pattern:** - -#### Step 1: Create Script in `scripts/` - -```python -# scripts/my_new_feature.py -from dotenv import load_dotenv -load_dotenv() # MUST BE FIRST LINE - -import argparse -from pathlib import Path -from snowddl_core import SnowflakeClient # Use OOP framework - -def main(): - """Main entry point for my new feature.""" - parser = argparse.ArgumentParser( - description="Description of my feature" - ) - parser.add_argument('--option', help='Example option') - args = parser.parse_args() - - # Implementation here - print(f"Running my feature with option: {args.option}") - -if __name__ == "__main__": - main() -``` - -**Critical Rules:** -- โœ… `load_dotenv()` MUST be the first line -- โœ… Use `argparse` for CLI interface -- โœ… Import from `snowddl_core` OOP framework -- โœ… Include docstrings -- โœ… Add type hints - -#### Step 2: Add Wrapper in `src/management_cli.py` +## Adding a New UV Command (4-Step Pattern) -```python -def my_new_feature(): - """Run my new feature command.""" - from my_new_feature import main - main() -``` +See CLAUDE.md "Creating New Commands" for the template. Summary: -#### Step 3: Register in `pyproject.toml` - -```toml -[project.scripts] -my-new-feature = "src.management_cli:my_new_feature" -``` +1. **Create script** in `scripts/my_script.py` - `load_dotenv()` MUST be first, use argparse +2. **Add wrapper** in `src/management_cli.py` +3. **Register** in `pyproject.toml` under `[project.scripts]` +4. **Test**: `uv sync && uv run my-command --help` -#### Step 4: Test the Command +## Running Tests ```bash -# Sync dependencies -uv sync - -# Test command -uv run my-new-feature --help -uv run my-new-feature --option test +uv run pytest -v # All tests +uv run pytest tests/test_foo.py -v # Specific file +uv run pytest --cov=src # With coverage +uv run pytest -n auto # Parallel (faster) +uv run pytest --lf # Only last-failed +uv run pytest -vv --tb=long # Detailed failure output +uv run pytest --pdb # Drop into debugger on failure ``` -### Writing Tests - -**Test Location Strategy:** -- `tests/test_.py` - Unit tests for `src/.py` -- `tests/integration/` - End-to-end tests -- `tests/manual/` - Complex manual verification scripts - -**Example Unit Test:** - +**Mocking Snowflake** (required for unit tests - no real connection needed): ```python -# tests/test_my_feature.py -"""Tests for my new feature.""" -import pytest -from unittest.mock import Mock, patch -from my_new_feature import main - -class TestMyFeature: - """Test suite for my feature.""" - - def test_basic_functionality(self): - """Test basic feature operation.""" - # Arrange - expected = "expected result" - - # Act - result = my_function() - - # Assert - assert result == expected - - @patch('my_new_feature.SnowflakeClient') - def test_with_mock_snowflake(self, mock_client): - """Test feature with mocked Snowflake connection.""" - # Setup mock - mock_client.return_value.execute.return_value = [] - - # Test with mock - result = main() - - # Verify - mock_client.return_value.execute.assert_called_once() -``` - -**Run Tests:** - -```bash -# Run all tests -uv run pytest -v - -# Run specific test file -uv run pytest tests/test_my_feature.py -v - -# Run with coverage -uv run pytest --cov=src --cov-report=term - -# Run in parallel (faster) -uv run pytest -n auto - -# Run only failed tests from last run -uv run pytest --lf -``` - ---- - -## ๐ŸŽจ Coding Standards - -### Python Style Guide - -**Follow these conventions:** - -1. **Type Hints (Required)** - ```python - def create_user( - username: str, - email: str, - user_type: UserType - ) -> dict[str, Any]: - """Create a new user.""" - pass - ``` - -2. **Docstrings (Required)** - ```python - def process_data(data: list[dict]) -> pd.DataFrame: - """ - Process raw data into structured DataFrame. - - Args: - data: List of dictionaries containing raw data - - Returns: - Pandas DataFrame with processed data - - Raises: - ValueError: If data is empty or malformed - """ - pass - ``` - -3. **Error Handling** - ```python - # Use custom exceptions - from snowddl_core.exceptions import SnowDDLError - - if not username: - raise SnowDDLError("Username is required") - ``` - -4. **Constants** - ```python - # Use uppercase for constants - DEFAULT_WAREHOUSE_SIZE = "X-SMALL" - MAX_PASSWORD_LENGTH = 256 - ``` - -### Pre-commit Checks - -**These run automatically on commit:** - -```yaml -# .pre-commit-config.yaml -- Black (code formatting) -- YAML validation -- Trailing whitespace removal -- Secrets detection (prevent credential leaks) -- File size limits -``` - -**Run manually:** - -```bash -# Check all files -uv run pre-commit run --all-files - -# Check specific files -uv run pre-commit run --files src/my_file.py - -# Skip hooks (rarely needed) -git commit --no-verify -``` - ---- - -## ๐Ÿ› Debugging Workflows - -### Debug a Failing Test - -```bash -# Run with detailed output -uv run pytest tests/test_user_management.py -vv --tb=long - -# Run with debugger -uv run pytest tests/test_user_management.py --pdb - -# Print test output (disable capture) -uv run pytest tests/test_user_management.py -s -``` - -### Debug a Command - -```python -# Add to script for debugging -import pdb; pdb.set_trace() # Breakpoint - -# Or use rich for better output -from rich import print -print(f"Debug info: {variable}") -``` - -### Common Issues - -**Issue: Import errors** -```bash -# Solution: Ensure sync is up to date -uv sync --all-extras --dev -``` - -**Issue: Test failures with Snowflake connection** -```bash -# Solution: Tests use mocks, no real connection needed -# Check your mock setup: @patch('module.SnowflakeClient') def test_with_mock(self, mock_client): - # Configure mock to return expected values mock_client.return_value.execute.return_value = [] + # test implementation ``` -**Issue: Pre-commit hooks failing** -```bash -# Solution: Auto-fix most issues -uv run pre-commit run --all-files - -# Then commit the fixes -git add . -git commit -m "fix: Apply pre-commit fixes" -``` - ---- - -## ๐Ÿ“ Pull Request Workflow - -### Creating a Feature Branch +## PR Workflow ```bash -# Always branch from the release branch (v0.2, v0.3, etc.) -git checkout v0.2 -git pull origin v0.2 -git checkout -b feature/my-awesome-feature -``` - -### Development Cycle +# 1. Branch from release branch +git checkout v0.2 && git pull +git checkout -b feature/my-feature -```bash -# 1. Make changes -vim src/my_file.py - -# 2. Write tests -vim tests/test_my_file.py - -# 3. Run quality checks +# 2. Develop + test uv run pre-commit run --all-files uv run pytest -v -# 4. Commit with conventional commit format -git add . -git commit -m "feat: Add my awesome feature" - -# 5. Push and create PR -git push -u origin feature/my-awesome-feature -gh pr create --base v0.2 --title "feat: Add my awesome feature" -``` - -### Commit Message Format +# 3. Commit (conventional commits) +git commit -m "feat: Add my feature" -**Use Conventional Commits:** - -``` -: - -[optional body] - -[optional footer] -``` - -**Types:** -- `feat:` - New feature -- `fix:` - Bug fix -- `docs:` - Documentation changes -- `test:` - Test changes -- `refactor:` - Code refactoring -- `chore:` - Maintenance tasks -- `ci:` - CI/CD changes - -**Examples:** - -```bash -git commit -m "feat: Add user bulk import command" -git commit -m "fix: Correct warehouse auto-suspend calculation" -git commit -m "docs: Update QUICKSTART with new commands" -git commit -m "test: Add integration tests for user management" +# 4. Push + create PR +git push -u origin feature/my-feature +gh pr create --base v0.2 ``` -### PR Checklist +**Commit types:** `feat:`, `fix:`, `docs:`, `test:`, `refactor:`, `chore:`, `ci:` -Before submitting, ensure: +## PR Checklist -- [ ] All tests pass: `uv run pytest -v` -- [ ] Pre-commit checks pass: `uv run pre-commit run --all-files` +- [ ] Tests pass: `uv run pytest -v` +- [ ] Pre-commit passes: `uv run pre-commit run --all-files` - [ ] New features have tests -- [ ] Documentation is updated -- [ ] Commit messages follow convention - [ ] No secrets in code -- [ ] Type hints are added -- [ ] Docstrings are complete - ---- - -## ๐Ÿงช Testing Best Practices - -### Test Organization - -```python -# tests/test_user_management.py -class TestUserCreation: - """Tests for user creation functionality.""" - - def setup_method(self): - """Set up test fixtures before each test.""" - self.manager = UserManager(config_directory=Path("/tmp/test")) - - def teardown_method(self): - """Clean up after each test.""" - shutil.rmtree("/tmp/test", ignore_errors=True) - - def test_create_person_user(self): - """Test creating a PERSON type user.""" - result = self.manager.create_user( - first_name="John", - last_name="Doe", - user_type=UserType.PERSON - ) - assert "JOHN_DOE" in result -``` - -### Mocking Snowflake Connections - -```python -# Always mock Snowflake for unit tests -@patch('snowflake.connector.connect') -def test_snowflake_operation(mock_connect): - """Test operation with mocked Snowflake connection.""" - # Setup mock - mock_cursor = Mock() - mock_cursor.fetchall.return_value = [('RESULT',)] - mock_connect.return_value.cursor.return_value = mock_cursor - - # Run test - result = my_snowflake_function() - - # Verify - assert result == [('RESULT',)] - mock_connect.assert_called_once() -``` - -### Property-Based Testing (Advanced) - -```python -from hypothesis import given, strategies as st - -@given( - username=st.text(min_size=1, max_size=50), - email=st.emails() -) -def test_user_creation_with_random_inputs(username, email): - """Test user creation with random inputs (property-based).""" - # This runs 100+ times with different random inputs - result = create_user(username, email) - assert validate_user(result) -``` - ---- - -## ๐Ÿ—๏ธ Using the SnowDDL OOP Framework - -### Creating Objects Programmatically - -```python -from snowddl_core import User, Role, Warehouse, WarehouseSize, UserType -from snowddl_core.safety import create_checkpoint - -# Create a user object -user = User( - name="JOHN_DOE", - comment="Data Analyst", - user_type=UserType.PERSON, - email="john@company.com", - default_role="ANALYST_ROLE" -) - -# Create a warehouse -warehouse = Warehouse( - name="ANALYTICS_WH", - size=WarehouseSize.SMALL, - auto_suspend=300, - auto_resume=True, - comment="Analytics team warehouse" -) - -# Create safety checkpoint before modifications -checkpoint = create_checkpoint("Before user creation") - -# Write to YAML -user.to_yaml(Path("snowddl/user.yaml")) -warehouse.to_yaml(Path("snowddl/warehouse.yaml")) -``` - -### Reading YAML Configurations - -```python -from user_management.yaml_handler import YAMLHandler - -# Read existing configuration -handler = YAMLHandler("snowddl/user.yaml") -users = handler.load() - -# Modify -users["NEW_USER"] = { - "comment": "New data analyst", - "type": "PERSON", - "email": "new@company.com" -} - -# Write back -handler.save(users) -``` - ---- - -## ๐Ÿ“Š Performance Testing - -### Benchmarking Commands - -```bash -# Run performance benchmarks -uv run pytest tests/ --benchmark-only - -# Compare benchmark results -uv run pytest tests/ --benchmark-compare -``` - -### Writing Benchmarks - -```python -def test_user_creation_performance(benchmark): - """Benchmark user creation performance.""" - result = benchmark( - create_user, - username="TEST_USER", - email="test@example.com" - ) - assert result is not None -``` - ---- - -## ๐Ÿ” Code Review Guidelines - -### As a Reviewer - -**Check for:** -- [ ] Tests cover new functionality -- [ ] No hardcoded credentials or secrets -- [ ] Type hints are present -- [ ] Docstrings are complete -- [ ] Error handling is appropriate -- [ ] No breaking changes without migration path -- [ ] CI checks pass -- [ ] Performance implications considered - -### As a Contributor - -**Before requesting review:** -- [ ] Self-review your code -- [ ] Remove debug statements and commented code -- [ ] Ensure variable names are descriptive -- [ ] Add comments for complex logic -- [ ] Update documentation -- [ ] Test edge cases - ---- - -## ๐Ÿšข Release Process - -### Pre-Release Checklist +- [ ] Commit messages follow convention -See `docs/releases/RELEASE_CHECKLIST.md` for full details. +## Pre-commit Hooks -**Critical steps:** +Runs automatically on commit: Black formatting, YAML validation, trailing whitespace, secrets detection, file size limits. ```bash -# 1. Run full test suite -uv run pytest -v - -# 2. All tests MUST pass -# No release with failing tests! - -# 3. Update version -vim pyproject.toml # Update version number - -# 4. Update changelog -vim CHANGELOG.md - -# 5. Create release PR -git checkout -b release/v0.3.0 -git commit -m "chore: Prepare v0.3.0 release" -gh pr create --base main - -# 6. After merge, tag release -git checkout main -git pull -git tag v0.3.0 -git push origin v0.3.0 # Triggers release workflow +uv run pre-commit run --all-files # Manual run +uv run pre-commit run --files src/my_file.py # Specific files ``` ---- - -## ๐ŸŽฏ Common Development Tasks - -### Add a New Snowflake Object Type - -1. **Create class in `src/snowddl_core/`** - ```python - # src/snowddl_core/account_objects.py - class MyNewObject(AccountLevelObject): - """New Snowflake object type.""" - pass - ``` - -2. **Add to exports in `__init__.py`** - -3. **Write tests** - -4. **Update documentation** - -### Add a New Authentication Method - -1. **Extend authentication module** - ```python - # src/user_management/authentication.py - class NewAuthMethod: - """New authentication method.""" - pass - ``` - -2. **Update user YAML schema** - -3. **Add tests with mocks** - -4. **Update security documentation** - -### Add a New Management Command - -See "Adding a New UV Command" section above. - ---- - -## ๐Ÿ“š Essential Documentation - -### For Developers - -- `CLAUDE.md` - Project overview and quick reference -- `docs/llm-context/PATTERNS.md` - Code patterns and conventions -- `docs/contributing/CONTRIBUTING.md` - Contribution guidelines -- `docs/contributing/HOW_TO_TEST.md` - Testing strategies -- `src/snowddl_core/README.md` - OOP framework documentation - -### For Understanding the Domain - -- `docs/guide/ARCHITECTURE.md` - System architecture -- `docs/guide/QUICKSTART.md` - User guide -- `docs/guide/SCHEMA_GRANTS.md` - Schema permission management - ---- - -## ๐Ÿ†˜ Getting Help - -### When Stuck +## Common Issues -1. **Check existing tests** for examples -2. **Read module docstrings** for API documentation -3. **Search GitHub issues** for similar problems -4. **Ask in discussions** or open an issue +| Problem | Fix | +|---------|-----| +| Import errors | `uv sync --all-extras --dev` | +| Tests need Snowflake | Tests use mocks - check mock setup | +| Pre-commit failing | Run `uv run pre-commit run --all-files`, commit fixes | -### Useful Commands +## Useful Commands ```bash -# See all UV commands -uv run snowtower - -# Get help for specific command -uv run manage-users --help - -# Run interactive Python with project imports -uv run python ->>> from snowddl_core import User ->>> help(User) - -# Check dependencies -uv tree - -# Update dependencies -uv lock --upgrade -``` - ---- - -## โœจ Best Practices Summary - -1. **Always use `load_dotenv()` first** in scripts -2. **Write tests** for all new features -3. **Use type hints** everywhere -4. **Mock Snowflake** connections in tests -5. **Follow conventional commits** format -6. **Run pre-commit** before pushing -7. **Create checkpoints** before destructive operations -8. **Document** public APIs with docstrings -9. **Branch from release branches** not main -10. **Keep PRs focused** on single features - ---- - -## ๐ŸŽ“ Advanced Topics - -### Dependency Injection for Testing - -```python -# Make functions testable by injecting dependencies -def process_users( - config_path: Path, - yaml_handler: Optional[YAMLHandler] = None -): - """Process users with optional injected handler for testing.""" - handler = yaml_handler or YAMLHandler(config_path) - # Use handler... +uv run --help # See all UV commands +uv tree # Check dependencies +uv lock --upgrade # Update dependencies +uv run python # Interactive Python with project imports ``` - -### Extending the CLI - -```python -# Add subcommands with click -import click - -@click.group() -def my_command(): - """My command group.""" - pass - -@my_command.command() -@click.option('--verbose', is_flag=True) -def subcommand(verbose): - """A subcommand.""" - if verbose: - print("Verbose mode enabled") -``` - -### Working with YAML Schema Validation - -```python -from pydantic import BaseModel, ValidationError - -class UserSchema(BaseModel): - """Schema for user YAML validation.""" - comment: str - type: str - email: str - -# Validate YAML data -try: - user = UserSchema(**yaml_data) -except ValidationError as e: - print(f"Invalid user config: {e}") -``` - ---- - -**Happy Coding! ๐Ÿš€** - -For questions or improvements to this skill, open an issue or PR on GitHub. diff --git a/.claude/skills/snowtower-maintainer/PROJECT_STRUCTURE.md b/.claude/skills/snowtower-maintainer/PROJECT_STRUCTURE.md deleted file mode 100644 index 48fd208..0000000 --- a/.claude/skills/snowtower-maintainer/PROJECT_STRUCTURE.md +++ /dev/null @@ -1,71 +0,0 @@ -# SnowTower Project Structure Reference - -Quick reference for the maintainer skill. - -## Directory Layout - -``` -snowtower/ -โ”œโ”€โ”€ .claude/ # Claude Code configuration -โ”‚ โ”œโ”€โ”€ skills/ # Skills (model-invoked capabilities) -โ”‚ โ”‚ โ””โ”€โ”€ snowtower-maintainer/ -โ”‚ โ”œโ”€โ”€ agents/ # Agent definitions -โ”‚ โ””โ”€โ”€ patterns/ # Reusable patterns -โ”œโ”€โ”€ .github/ # GitHub configuration -โ”‚ โ”œโ”€โ”€ workflows/ # CI/CD workflows -โ”‚ โ”‚ โ”œโ”€โ”€ ci.yml # PR validation -โ”‚ โ”‚ โ”œโ”€โ”€ labeler.yml # Auto-labeling -โ”‚ โ”‚ โ”œโ”€โ”€ changelog.yml # Changelog generation -โ”‚ โ”‚ โ””โ”€โ”€ release.yml # Release automation -โ”‚ โ”œโ”€โ”€ ISSUE_TEMPLATE/ # Issue templates -โ”‚ โ””โ”€โ”€ PULL_REQUEST_TEMPLATE.md -โ”œโ”€โ”€ snowddl/ # SnowDDL YAML configurations -โ”‚ โ”œโ”€โ”€ user.yaml # User accounts -โ”‚ โ”œโ”€โ”€ business_role.yaml # Business roles -โ”‚ โ”œโ”€โ”€ tech_role.yaml # Technical roles -โ”‚ โ”œโ”€โ”€ warehouse.yaml # Warehouses -โ”‚ โ”œโ”€โ”€ *_policy.yaml # Security policies -โ”‚ โ””โ”€โ”€ {DATABASE}/ # Database-specific configs -โ”œโ”€โ”€ src/ # Python source code -โ”‚ โ”œโ”€โ”€ snowddl_core/ # OOP framework -โ”‚ โ”œโ”€โ”€ user_management/ # User lifecycle -โ”‚ โ”œโ”€โ”€ web/ # Streamlit apps -โ”‚ โ””โ”€โ”€ management_cli.py # CLI entry points -โ”œโ”€โ”€ scripts/ # Standalone scripts -โ”œโ”€โ”€ tests/ # Test suite -โ”œโ”€โ”€ docs/ # Documentation -โ”‚ โ”œโ”€โ”€ guide/ # User guides -โ”‚ โ”œโ”€โ”€ agents/ # Agent documentation -โ”‚ โ””โ”€โ”€ releases/ # Release documentation -โ”œโ”€โ”€ pyproject.toml # Project configuration -โ”œโ”€โ”€ README.md # Main documentation -โ””โ”€โ”€ CLAUDE.md # Claude Code instructions -``` - -## Key Files to Monitor - -| File | Purpose | Update Frequency | -|------|---------|------------------| -| `README.md` | Main project documentation | After features/releases | -| `CLAUDE.md` | Claude Code instructions | When patterns change | -| `pyproject.toml` | Commands and dependencies | When adding commands | -| `.claude/agents/*.md` | Agent definitions | When capabilities change | -| `docs/guide/*.md` | User guides | When workflows change | - -## Current Statistics (Update These) - -```yaml -# Last updated: 2024-XX-XX -users: ~13 -databases: ~6 -warehouses: ~8 -agents: ~20+ -workflows: 4 -``` - -## Maintenance Schedule - -- **Weekly**: Check for broken links, outdated statistics -- **Per Release**: Full documentation audit -- **Per Feature**: Update relevant docs and README -- **Quarterly**: Agent consolidation review diff --git a/.claude/skills/snowtower-maintainer/SKILL.md b/.claude/skills/snowtower-maintainer/SKILL.md index 0191e20..d3cbc48 100644 --- a/.claude/skills/snowtower-maintainer/SKILL.md +++ b/.claude/skills/snowtower-maintainer/SKILL.md @@ -5,257 +5,81 @@ description: Maintains SnowTower project documentation, README, and Claude confi # SnowTower Project Maintainer -A specialized skill for maintaining the SnowTower project's documentation, README, and Claude Code configuration. +Assumes CLAUDE.md is loaded for project context. -## Core Responsibilities +## README Audit Checklist -### 1. README Maintenance - -Keep `README.md` accurate and current: - -- **Version badges**: Ensure CI/CD badges point to correct workflows -- **Command references**: Verify all `uv run` commands are valid -- **Architecture diagrams**: Keep mermaid diagrams in sync with actual structure -- **Statistics**: Update user counts, database counts, warehouse counts -- **Links**: Verify all internal links resolve correctly - -**Audit checklist:** -```bash -# Verify commands mentioned in README actually exist -uv run --help | grep -E "snowddl-plan|deploy-safe|manage-users" - -# Check workflow badge URLs match actual workflow files -ls .github/workflows/ - -# Verify documentation links -find docs/ -name "*.md" | head -20 -``` - -### 2. Claude Folder Maintenance - -Maintain `.claude/` organization: - -``` -.claude/ -โ”œโ”€โ”€ skills/ # Claude Code skills (like this one) -โ”œโ”€โ”€ agents/ # Agent definitions for task delegation -โ”œโ”€โ”€ patterns/ # Reusable patterns and templates -โ””โ”€โ”€ settings.local.json -``` - -**Agent audit tasks:** -- Remove duplicate or redundant agents -- Consolidate agents with overlapping purposes -- Update agent descriptions to match current capabilities -- Ensure agents reference correct file paths - -**Pattern audit tasks:** -- Verify patterns match current project conventions -- Update code examples in patterns -- Remove outdated patterns - -### 3. Documentation Sync +1. **Verify commands** mentioned in README exist: + ```bash + uv run --help | grep -E "snowddl-plan|deploy-safe|manage-users" + ``` +2. **Check workflow badges** match `.github/workflows/` +3. **Verify documentation links** resolve to existing files +4. **Update statistics** (users, databases, warehouses): + ```bash + echo "Users: $(grep -c '^ [A-Z]' snowddl/user.yaml 2>/dev/null || echo N/A)" + echo "Databases: $(ls -d snowddl/*/ 2>/dev/null | grep -v __pycache__ | wc -l)" + echo "Warehouses: $(grep -c '^ [A-Z]' snowddl/warehouse.yaml 2>/dev/null || echo N/A)" + ``` -Ensure docs reflect actual project state: +## Documentation Sync Table | Doc File | Should Match | |----------|--------------| | `docs/guide/MANAGEMENT_COMMANDS.md` | `pyproject.toml` scripts | | `docs/guide/QUICKSTART.md` | Current setup process | | `docs/guide/SCHEMA_GRANTS.md` | Current grant handling | -| Agent files in `.claude/agents/` | Available functionality | - -## Maintenance Procedures - -### Quick Health Check - -```bash -# 1. Verify project structure -ls -la snowddl/ src/ scripts/ docs/ - -# 2. Check available commands -uv run --help - -# 3. Verify tests pass -uv run pytest --co -q | tail -5 - -# 4. Check pre-commit status -uv run pre-commit run --all-files -``` - -### README Update Workflow - -1. **Gather current state:** - ```bash - # Count configured users - grep -c "^ [A-Z]" snowddl/user.yaml - - # Count databases - ls -d snowddl/*/ | grep -v __pycache__ | wc -l - - # List warehouses - grep "^ [A-Z]" snowddl/warehouse.yaml - ``` - -2. **Verify commands:** - ```bash - # Extract commands from pyproject.toml - grep -A1 "\[project.scripts\]" pyproject.toml - ``` - -3. **Update statistics section** in README with current counts - -4. **Verify all links** resolve to existing files - -### Agent Consolidation - -When auditing `.claude/agents/`: - -1. **List all agents:** - ```bash - ls .claude/agents/*.md - ``` - -2. **Identify overlaps:** Look for agents with similar purposes - -3. **Consolidation criteria:** - - Merge agents that serve the same domain - - Keep agents with distinct, valuable roles - - Remove agents that duplicate built-in capabilities - -4. **Update references:** After consolidation, update any docs referencing removed agents - -### Self-Maintenance - -This skill should maintain itself by: - -1. Keeping this SKILL.md up to date with project changes -2. Adding new maintenance procedures as project evolves -3. Updating file paths when project structure changes -4. Documenting new patterns discovered during maintenance - -## Common Maintenance Tasks - -### Task: Update README Statistics - -```markdown -### Status & Metrics - -- **Active Users**: [COUNT] configured users with MFA -- **Databases**: [COUNT] production databases managed -- **Warehouses**: [COUNT] warehouses with auto-suspend -``` - -Update these by running: -```bash -echo "Users: $(grep -c '^ [A-Z]' snowddl/user.yaml)" -echo "Databases: $(ls -d snowddl/*/ 2>/dev/null | grep -v __pycache__ | wc -l)" -echo "Warehouses: $(grep -c '^ [A-Z]' snowddl/warehouse.yaml)" -``` +| `README.md` | Actual project capabilities | -### Task: Verify Workflow Badges +## Quick Health Check -Check that README badges match actual workflows: ```bash -# List workflows -ls .github/workflows/ - -# Verify badge URLs in README reference these files -grep "actions/workflows" README.md +ls -la snowddl/ src/ scripts/ docs/ # Verify structure +uv run --help # Check commands +uv run pytest --co -q | tail -5 # Verify tests collect +uv run pre-commit run --all-files # Check formatting ``` -### Task: Audit Agent Definitions - -```bash -# List agents and their purposes -for f in .claude/agents/*.md; do - echo "=== $f ===" - head -5 "$f" - echo -done -``` - -### Task: Clean Up Obsolete Content - -Remove references to: -- Deleted files or directories -- Deprecated commands -- Old workflow names -- Removed features - -## Integration with Project - -This skill works with: - -- **CI/CD workflows**: `.github/workflows/` -- **SnowDDL configs**: `snowddl/*.yaml` -- **Python tooling**: `src/`, `scripts/` -- **Documentation**: `docs/` -- **Claude config**: `.claude/` - ## Release Planning -### Identify What Goes in a Release - 1. **Check open issues:** ```bash gh issue list --state open --label P1 gh issue list --state open ``` - -2. **Review workspace roadmap:** - - Check `/Users/ssciortino/Projects/snowtower-workspace/CLAUDE.md` for: - - "Strategic Recommendations & Next Steps" section - - Priority 1/2/3 items - - Technical debt to address - -3. **Gather completed work since last release:** +2. **Review completed work since last release:** ```bash - git log v0.X..main --oneline + git log v0.X..HEAD --oneline + ``` +3. **Create version branch:** + ```bash + git checkout main && git pull + git checkout -b v0.X + git push -u origin v0.X + ``` +4. **Pre-release checklist:** + - [ ] `uv run pytest` passes + - [ ] CHANGELOG.md updated + - [ ] README current (badges, commands, stats) + - [ ] Documentation links valid +5. **Release:** + ```bash + gh release create vX.Y --title "vX.Y - Title" --notes "..." ``` -### Create Version Branch +## .claude/ Directory Structure -```bash -# Create and push version branch -git checkout main -git pull origin main -git checkout -b v0.X -git push -u origin v0.X ``` - -### Pre-Release Checklist - -1. **Run full test suite:** `uv run pytest` -2. **Update CHANGELOG.md** with new features -3. **Verify README is current** (badges, commands, stats) -4. **Check documentation links** resolve correctly -5. **Create PR to main** with release summary - -### Release Workflow - -```bash -# After PR is merged -git checkout main -git pull -gh release create vX.Y --title "vX.Y - Title" --notes "..." +.claude/ +โ”œโ”€โ”€ patterns/ # Reusable patterns (e.g., SERVICE_ACCOUNT_CREATION_PATTERN.md) +โ”œโ”€โ”€ skills/ # Claude Code skills (4 task-specific guides) +โ””โ”€โ”€ settings.local.json ``` -### Version Naming - -- **v0.X**: Pre-1.0 development releases -- **v1.0**: First stable release (target: installable CLI) -- **vX.Y.Z**: Semantic versioning post-1.0 - ## When to Trigger -Invoke this skill when: -- User asks to "update the README" -- User mentions "documentation maintenance" -- User wants to "audit the .claude folder" -- User asks about "project documentation" -- After significant feature additions +- User asks to update README or documentation - Before releases to ensure docs are current +- After significant feature additions - When onboarding new contributors -- **User asks to plan a new release version** -- **User wants to start work on vX.Y** +- User asks to plan a new release version diff --git a/.claude/skills/snowtower-user/SKILL.md b/.claude/skills/snowtower-user/SKILL.md index 80fa3d1..636bb0d 100644 --- a/.claude/skills/snowtower-user/SKILL.md +++ b/.claude/skills/snowtower-user/SKILL.md @@ -5,86 +5,37 @@ description: Helps end-users get Snowflake access and use the platform. Use when # SnowTower End-User Guide -A skill for helping end-users navigate the SnowTower platform to get Snowflake access and start working with data. +Assumes CLAUDE.md is loaded for project context. -## Who This Skill Is For +## Getting Access (3 Steps) -- **Data analysts** who need to query Snowflake data -- **Data scientists** who need database access for analysis -- **Engineers** who need to connect applications to Snowflake -- **New team members** requesting their first Snowflake account - -## Quick Reference - -### Getting Access (3 Steps) - -``` -Step 1: Generate RSA Keys โ†’ Step 2: Submit Request โ†’ Step 3: Connect - (on your machine) (GitHub issue) (after approval) -``` - ---- - -## Step 1: Generate Your RSA Keys - -**You MUST do this BEFORE requesting access.** +### Step 1: Generate RSA Keys ```bash -# Generate RSA key pair (run on your local machine) +# Generate key pair openssl genrsa 2048 | openssl pkcs8 -topk8 -inform PEM -nocrypt -out ~/.ssh/snowflake_rsa_key.p8 openssl rsa -in ~/.ssh/snowflake_rsa_key.p8 -pubout -out ~/.ssh/snowflake_rsa_key.pub -# Secure your private key (IMPORTANT!) +# Secure private key chmod 400 ~/.ssh/snowflake_rsa_key.p8 -# Display your PUBLIC key (copy this for the access request) +# Copy public key for access request cat ~/.ssh/snowflake_rsa_key.pub ``` -**Output looks like:** -``` ------BEGIN PUBLIC KEY----- -MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA... -...many lines of characters... ------END PUBLIC KEY----- -``` - -### Key Security Rules - -| Key Type | File | Share? | -|----------|------|--------| -| **Private key** | `~/.ssh/snowflake_rsa_key.p8` | **NEVER share this** | -| **Public key** | `~/.ssh/snowflake_rsa_key.pub` | Safe to share | - ---- +**Private key** (`~/.ssh/snowflake_rsa_key.p8`) - NEVER share. **Public key** (`.pub`) - safe to share. -## Step 2: Request Access +### Step 2: Submit Access Request -1. Go to the **[Access Request Form](../../issues/new/choose)** +1. Go to the [Access Request Form](../../issues/new/choose) 2. Select "New User Request" -3. Fill in your details: - - Full name - - Email address - - Team/department - - Reason for access - - **Paste your PUBLIC key** (from Step 1) -4. Submit the form - -**Typical approval time:** 3-5 business days - ---- - -## Step 3: Connect to Snowflake - -After your account is approved, you'll receive: -- Your **username** (usually FIRSTNAME_LASTNAME) -- The **account identifier** -- Your **default role** and **warehouse** +3. Fill in: name, email, team, reason, **paste public key** +4. Typical approval: 3-5 business days -### Using Snow CLI (Recommended) +### Step 3: Connect to Snowflake +**Snow CLI (recommended):** ```bash -# Add your connection snow connection add \ --connection-name prod \ --account YOUR_ACCOUNT \ @@ -92,163 +43,39 @@ snow connection add \ --authenticator SNOWFLAKE_JWT \ --private-key-path ~/.ssh/snowflake_rsa_key.p8 -# Test the connection snow sql -c prod -q "SELECT CURRENT_USER(), CURRENT_ROLE()" ``` -### Using Python - +**Python:** ```python import snowflake.connector - conn = snowflake.connector.connect( account='YOUR_ACCOUNT', user='YOUR_USERNAME', - private_key_file_pwd=None, private_key_file='~/.ssh/snowflake_rsa_key.p8', warehouse='MAIN_WAREHOUSE', role='YOUR_ROLE' ) ``` -### Using the Snowflake Web UI - -1. Go to your organization's Snowflake URL -2. Enter your username -3. Use the **password provided by IT** (not your RSA key) -4. Enable MFA when prompted - ---- - -## What You Get After Approval - -### Your Default Role - -New users typically receive a role like `SNOWTOWER_USERS__T_ROLE` which grants: -- Read access to shared production data -- Access to common warehouses -- Ability to create objects in your personal database - -### Your Personal Database - -You get your own database: `DEV_YOURNAME` - -```sql --- Switch to your database -USE DATABASE DEV_YOURNAME; - --- Create schemas and tables freely -CREATE SCHEMA my_analysis; -CREATE TABLE my_analysis.test_data (id INT, value VARCHAR); -``` - -### Your Default Warehouse - -Usually `MAIN_WAREHOUSE`: -- Auto-suspends after 60 seconds of inactivity -- X-Small size by default -- Shared resource - be mindful of heavy queries - ---- - ## First Session Checklist ```sql --- 1. Check your current context SELECT CURRENT_USER(), CURRENT_ROLE(), CURRENT_WAREHOUSE(); - --- 2. See what databases you can access SHOW DATABASES; - --- 3. See what roles you have SHOW ROLES; - --- 4. Switch to your dev database USE DATABASE DEV_YOURNAME; - --- 5. Create your first schema CREATE SCHEMA IF NOT EXISTS sandbox; -USE SCHEMA sandbox; - --- 6. Test creating a table -CREATE TABLE test (id INT); -INSERT INTO test VALUES (1), (2), (3); -SELECT * FROM test; -DROP TABLE test; -``` - ---- - -## Common Issues & Solutions - -### "Authentication failed" - -**Cause:** RSA key mismatch or incorrect setup - -**Solution:** -```bash -# Verify your private key is readable -ls -la ~/.ssh/snowflake_rsa_key.p8 - -# Check permissions (should be 400 or 600) -chmod 400 ~/.ssh/snowflake_rsa_key.p8 - -# Verify the public key matches what was submitted -cat ~/.ssh/snowflake_rsa_key.pub -``` - -### "Insufficient privileges" - -**Cause:** You don't have access to that object - -**Solution:** -- Check you're using the correct role: `SELECT CURRENT_ROLE();` -- Request additional access if needed via GitHub issue - -### "Warehouse is suspended" - -**Cause:** Warehouse auto-suspended to save costs - -**Solution:** -```sql --- Just run a query - it auto-resumes -SELECT 1; ``` -### "Cannot connect to Snowflake" +## Troubleshooting -**Checklist:** -1. Is your account approved? (Check the GitHub issue) -2. Is the account identifier correct? -3. Is your private key path correct? -4. Are you on the corporate network / VPN if required? - ---- +| Problem | Fix | +|---------|-----| +| Authentication failed | Check key permissions: `chmod 400 ~/.ssh/snowflake_rsa_key.p8` | +| Insufficient privileges | Verify role: `SELECT CURRENT_ROLE();` - request more access via GitHub issue | +| Cannot connect | Check: account approved? Account ID correct? Key path correct? On VPN? | ## Getting More Access -Need access to additional databases, schemas, or roles? - -1. Open a new GitHub issue -2. Specify exactly what you need access to -3. Include business justification -4. Your request will be reviewed by an admin - ---- - -## Two Authentication Methods - -| Method | Use For | How | -|--------|---------|-----| -| **RSA Key** | CLI, scripts, applications | Private key file | -| **Password** | Web UI only | Provided by IT | - -**Best Practice:** Always use RSA key authentication for programmatic access. Only use password for the web interface. - ---- - -## Need Help? - -- **Access issues:** Open a GitHub issue -- **Connection problems:** Check the troubleshooting section above -- **General questions:** Ask your team lead or Snowflake admin +Open a GitHub issue specifying what you need and business justification. diff --git a/CLAUDE.md b/CLAUDE.md index 8f44ef0..e3b8a3d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -8,10 +8,12 @@ # Essential commands uv run snowddl-plan # Preview infrastructure changes uv run snowddl-apply # Apply changes (needs confirmation) +uv run deploy-safe # Safe deployment (preserves schema grants) uv run manage-users # User management uv run manage-warehouses # Warehouse management uv run manage-costs # Cost analysis uv sync # Install dependencies +uv run pytest # Run tests ``` ## Critical Rules @@ -41,11 +43,11 @@ snowtower-snowddl/ โ”‚ โ”œโ”€โ”€ snowddl_core/ # OOP framework โ”‚ โ””โ”€โ”€ management_cli.py # CLI entry points โ”œโ”€โ”€ scripts/ # Management scripts +โ”œโ”€โ”€ tests/ # Test suite โ”œโ”€โ”€ docs/ # Documentation โ”‚ โ”œโ”€โ”€ guide/ # User guides โ”‚ โ”œโ”€โ”€ contributing/ # Developer docs -โ”‚ โ”œโ”€โ”€ llm-context/ # LLM configuration -โ”‚ โ””โ”€โ”€ ... +โ”‚ โ””โ”€โ”€ releases/ # Release docs โ””โ”€โ”€ pyproject.toml # UV/Python config ``` @@ -104,6 +106,47 @@ The `--env-prefix` parameter adds a prefix to all object names (databases, schem **Must use `-r ACCOUNTADMIN`** to see all grants across roles/schemas when running plan. +### SnowDDL Object Type Reference + +| Object Type | Grant Key | Example | +|------------|-----------|---------| +| Database | `DATABASE:` | `DATABASE:USAGE,CREATE SCHEMA` | +| Schema | `SCHEMA:` | `SCHEMA:USAGE,CREATE TABLE` | +| Table | `TABLE:` | `TABLE:SELECT,INSERT` | +| View | `VIEW:` | `VIEW:SELECT` | +| Warehouse | `WAREHOUSE:` | `WAREHOUSE:USAGE,OPERATE` | +| Stage | `STAGE:` | `STAGE:USAGE,READ,WRITE` | +| Function | `FUNCTION:` | `FUNCTION:USAGE` | +| Procedure | `PROCEDURE:` | `PROCEDURE:USAGE` | + +### Role Types and Naming + +SnowDDL automatically creates roles with specific suffixes: + +| Role Type | Suffix | Purpose | Example | +|-----------|--------|---------|---------| +| User Role | `__U_ROLE` | Per-user role | `DLT__U_ROLE` | +| Technical Role | `__T_ROLE` | Service/app permissions | `DLT_STRIPE_TECH_ROLE__T_ROLE` | +| Business Role | `__B_ROLE` | Logical groupings | `DLT_STRIPE_ROLE__B_ROLE` | +| Schema Owner | `__S_ROLE` | Object ownership | `SOURCE_STRIPE__STRIPE_WHY__OWNER__S_ROLE` | +| Database Owner | `__D_ROLE` | Database-level ownership | `SOURCE_STRIPE__OWNER__D_ROLE` | + +### Role Hierarchy (RBAC) + +``` +ACCOUNTADMIN (top-level) + โ†“ +SYSADMIN โ†’ USERADMIN โ†’ SECURITYADMIN + โ†“ โ†“ โ†“ +Business Roles (__B_ROLE) + โ†“ +Technical Roles (__T_ROLE) + โ†“ +Object Permissions +``` + +**Inheritance**: User โ†’ User Role (__U_ROLE) โ†’ Business Role (__B_ROLE) โ†’ Technical Role (__T_ROLE) + Schema Owner (__S_ROLE) + ### Schema Grants in SnowDDL Schema grants are defined in `tech_role.yaml` using `SCHEMA:` format: @@ -112,13 +155,11 @@ Schema grants are defined in `tech_role.yaml` using `SCHEMA:` format: # tech_role.yaml DBT_STRIPE_ROLE: grants: - # Schema-level grants (the key insight!) SCHEMA:USAGE: - SOURCE_STRIPE.STRIPE_WHY - PROJ_STRIPE.PROJ_STRIPE SCHEMA:CREATE TABLE,CREATE VIEW,MODIFY: - PROJ_STRIPE.PROJ_STRIPE - # Database grants DATABASE:USAGE,CREATE SCHEMA: - SOURCE_STRIPE - PROJ_STRIPE @@ -157,6 +198,7 @@ The "schema drift" issue occurs when: - [Schema Config](https://docs.snowddl.com/basic/yaml-configs/schema) - Schema params.yaml structure - [Technical Roles](https://docs.snowddl.com/basic/yaml-configs/technical-role) - Grant definitions - [Business Roles](https://docs.snowddl.com/basic/yaml-configs/business-role) - schema_read/write abstractions +- [Permission Model](https://docs.snowddl.com/basic/yaml-configs/permission-model) - Permission models - [Env Prefix Guide](https://docs.snowddl.com/guides/other-guides/env-prefix) - Environment separation ## Setting Up dbt Projects (CRITICAL) @@ -203,15 +245,6 @@ GRANT OWNERSHIP ON ALL VIEWS IN SCHEMA . TO ROLE ____OWNER__S_ROLE COPY CURRENT GRANTS; ``` -**Real example for PROJ_STRIPE.PROJ_STRIPE:** -```sql -GRANT OWNERSHIP ON ALL TABLES IN SCHEMA PROJ_STRIPE.PROJ_STRIPE - TO ROLE PROJ_STRIPE__PROJ_STRIPE__OWNER__S_ROLE COPY CURRENT GRANTS; - -GRANT OWNERSHIP ON ALL VIEWS IN SCHEMA PROJ_STRIPE.PROJ_STRIPE - TO ROLE PROJ_STRIPE__PROJ_STRIPE__OWNER__S_ROLE COPY CURRENT GRANTS; -``` - ### Why This Works The grant chain after setup: @@ -232,9 +265,49 @@ SQL access control error: View 'MY_VIEW' already exists, but current role has no **Fix:** Complete all 3 steps above. +## DLT (Data Loading) Permission Patterns + +### The Table Stage Problem + +When DLT loads data, it uses `COPY INTO` with table stages. **Only the TABLE OWNER can access table stages.** If DLT doesn't own the tables, loading fails with: + +``` +SQL access control error: Insufficient privileges to operate on table stage 'CHARGE' +``` + +### Solution: Grant Schema Owner Role to Business Role + +```yaml +# snowddl/business_role.yaml +DLT_STRIPE_ROLE: + comment: DLT Stripe pipeline - includes schema owner for table stage access + tech_roles: + - DLT_STRIPE_TECH_ROLE + schema_owner: + - SOURCE_STRIPE.STRIPE_WHY # Grants SOURCE_STRIPE__STRIPE_WHY__OWNER__S_ROLE + warehouse_usage: + - DLT +``` + +**Why this works**: `schema_owner` grants the auto-created `__OWNER__S_ROLE` to the business role. The user inherits through: User Role โ†’ Business Role โ†’ Schema Owner Role โ†’ table stage access. + +**What doesn't work**: Technical roles don't support `schema_owner`. You must use business roles. + +## Environment Variables + +Required in `.env`: +``` +SNOWFLAKE_ACCOUNT=your_account +SNOWFLAKE_USER=your_user +SNOWFLAKE_PASSWORD=your_password # Or use RSA key +SNOWFLAKE_ROLE=ACCOUNTADMIN +SNOWFLAKE_WAREHOUSE=COMPUTE_WH +FERNET_KEY=your_fernet_key # For password encryption +``` + ## For More Details -- [Full LLM Instructions](docs/llm-context/CLAUDE.md) -- [Project Context](docs/llm-context/CONTEXT.md) -- [Code Patterns](docs/llm-context/PATTERNS.md) - [CLI Reference](docs/guide/MANAGEMENT_COMMANDS.md) +- [Troubleshooting](docs/guide/TROUBLESHOOTING.md) +- [Architecture](docs/guide/ARCHITECTURE.md) +- [Service Account Pattern](.claude/patterns/SERVICE_ACCOUNT_CREATION_PATTERN.md) diff --git a/docs/README.md b/docs/README.md index 37eff3a..23ea5a3 100644 --- a/docs/README.md +++ b/docs/README.md @@ -10,8 +10,6 @@ | Fix schema permissions | [Schema Grants](guide/SCHEMA_GRANTS.md) | | Troubleshoot issues | [Troubleshooting](guide/TROUBLESHOOTING.md) | | View the changelog | [Changelog](releases/CHANGELOG.md) | -| Use LLMs with this repo | [LLM Context](llm-context/) | - --- ## Guide @@ -41,17 +39,6 @@ All user-facing documentation. --- -## LLM Context - -Configuration files for using LLMs with this codebase. - -- [README](llm-context/README.md) - Overview and quick start -- [CLAUDE.md](llm-context/CLAUDE.md) - Full project instructions for Claude -- [CONTEXT.md](llm-context/CONTEXT.md) - Domain knowledge and project context -- [PATTERNS.md](llm-context/PATTERNS.md) - Code patterns and conventions - ---- - ## Contributing For developers and contributors. diff --git a/docs/llm-context/CLAUDE.md b/docs/llm-context/CLAUDE.md deleted file mode 100644 index 42af253..0000000 --- a/docs/llm-context/CLAUDE.md +++ /dev/null @@ -1,329 +0,0 @@ -# CLAUDE.md - SnowTower SnowDDL Project Instructions - -## Project Overview - -**SnowTower SnowDDL** is an enterprise Snowflake infrastructure management platform using Infrastructure as Code (IaC) with SnowDDL. - -### What This Project Does -- Manages Snowflake infrastructure via declarative YAML files -- Provides CLI commands for user, warehouse, and cost management -- Includes security policies, network policies, and MFA compliance - -## Critical Rules - -### Always Do -1. **Use UV for all Python operations**: `uv run `, `uv sync`, `uv add ` -2. **Load environment variables**: Every script must start with `load_dotenv()` -3. **Run plan before apply**: Always `uv run snowddl-plan` before `uv run snowddl-apply` -4. **Follow existing patterns**: Check `scripts/` for examples before creating new commands - -### Never Do -1. **Never run `snowddl-apply` without user confirmation** -2. **Never commit credentials or secrets** -3. **Never modify `snowddl/*.yaml` without understanding the impact** -4. **Never use pip directly** - always use UV - -## Key Commands - -```bash -# Infrastructure -uv run snowddl-plan # Preview changes (ALWAYS RUN FIRST) -uv run snowddl-apply # Apply changes (REQUIRES CONFIRMATION) -uv run deploy-safe # Recommended: deploys + applies schema grants - -# User Management -uv run manage-users # Full user management CLI - -# Operations -uv run manage-warehouses # Warehouse management -uv run manage-costs # Cost analysis -uv run monitor-health # System health - -# Development -uv sync # Install dependencies -uv run pytest # Run tests -``` - -## Project Structure - -``` -snowtower-snowddl/ -โ”œโ”€โ”€ snowddl/ # Infrastructure definitions (YAML) -โ”‚ โ”œโ”€โ”€ user.yaml # User definitions -โ”‚ โ”œโ”€โ”€ role.yaml # Role definitions -โ”‚ โ”œโ”€โ”€ warehouse.yaml # Warehouse configs -โ”‚ โ”œโ”€โ”€ network_policy.yaml # Network policies -โ”‚ โ””โ”€โ”€ {DATABASE}/ # Database-specific configs -โ”œโ”€โ”€ src/ -โ”‚ โ”œโ”€โ”€ snowddl_core/ # OOP framework -โ”‚ โ”œโ”€โ”€ user_management/ # User lifecycle -โ”‚ โ””โ”€โ”€ management_cli.py # CLI entry points -โ”œโ”€โ”€ scripts/ # Management scripts -โ”œโ”€โ”€ docs/ # Documentation -โ””โ”€โ”€ pyproject.toml # UV/Python config -``` - -## Creating New Commands - -Follow this pattern: - -### 1. Create Script in `scripts/` - -```python -#!/usr/bin/env python3 -"""Description of what this script does.""" - -from dotenv import load_dotenv -load_dotenv() # MANDATORY - must be first - -import argparse -from src.snowddl_core import SnowDDLClient - -def main(): - parser = argparse.ArgumentParser(description="Command description") - parser.add_argument("--option", help="Option description") - args = parser.parse_args() - - # Implementation here - -if __name__ == "__main__": - main() -``` - -### 2. Add Wrapper in `src/management_cli.py` - -```python -def my_command(): - """Run my command.""" - from scripts.my_script import main - main() -``` - -### 3. Register in `pyproject.toml` - -```toml -[project.scripts] -my-command = "src.management_cli:my_command" -``` - -### 4. Test - -```bash -uv sync -uv run my-command --help -``` - -## YAML Configuration Patterns - -### User Definition -```yaml -# snowddl/user.yaml -ANALYST_USER: - type: PERSON - default_role: ANALYST_ROLE - default_warehouse: ANALYST_WH - must_change_password: false - network_policy: OFFICE_NETWORK_POLICY -``` - -### Role Definition -```yaml -# snowddl/role.yaml -ANALYST_ROLE: - comment: "Read-only analyst access" - grants: - - database: ANALYTICS - privileges: [USAGE] -``` - -### Service Account (RSA Key Auth) -```yaml -# snowddl/user.yaml -DBT_SERVICE: - type: SERVICE - default_role: DBT_ROLE - rsa_public_key: | - -----BEGIN PUBLIC KEY----- - MIIBIjANBgk... - -----END PUBLIC KEY----- -``` - -## Security Considerations - -### Authentication Hierarchy -1. **RSA Key Pairs** - Preferred for all users -2. **Encrypted Passwords** - Fallback only (Fernet encryption) -3. **MFA** - Required for human users by 2026 - -### Network Policies -- Human users: Restricted to office IP -- Service accounts: Typically unrestricted -- Emergency access: STEPHEN_RECOVERY has no network policy - -## Common Tasks - -### Add a New User -```bash -uv run manage-users create # Interactive wizard -uv run snowddl-plan # Preview -uv run deploy-safe # Apply (with schema grants) -``` - -### Check System Health -```bash -uv run monitor-health -uv run manage-costs --analyze -``` - -### Debug SnowDDL Issues -```bash -uv run snowddl-plan --verbose -# Check snowddl/*.yaml for syntax errors -``` - -## Environment Variables - -Required in `.env`: -``` -SNOWFLAKE_ACCOUNT=your_account -SNOWFLAKE_USER=your_user -SNOWFLAKE_PASSWORD=your_password # Or use RSA key -SNOWFLAKE_ROLE=ACCOUNTADMIN -SNOWFLAKE_WAREHOUSE=COMPUTE_WH -FERNET_KEY=your_fernet_key # For password encryption -``` - -## SnowDDL Deep Knowledge - -### Critical: Understanding `--env-prefix` - -**The `--env-prefix` parameter is NOT for environment variable naming!** - -```bash -# WRONG understanding - This does NOT read from SNOWFLAKE_* env vars -snowddl --env-prefix SNOWFLAKE plan - -# What it ACTUALLY does - Prefixes all OBJECT NAMES -# It looks for objects like: SNOWFLAKE__MY_DATABASE, SNOWFLAKE__MY_ROLE, etc. -``` - -The `--env-prefix` adds a prefix to object names for environment separation (DEV__, PROD__, etc.). -SnowDDL reads connection credentials from environment variables automatically (SNOWFLAKE_ACCOUNT, SNOWFLAKE_USER, etc.) without needing `--env-prefix`. - -**Correct usage**: -```bash -# Use -r ACCOUNTADMIN to see all grants when planning -snowddl -c snowddl -r ACCOUNTADMIN plan - -# For environment separation (multiple envs on same account) -snowddl -c snowddl --env-prefix DEV plan # Creates DEV__DATABASE, DEV__ROLE, etc. -``` - -### Schema Grants in Technical Roles - -Schema grants must be defined in `tech_role.yaml` using `SCHEMA:` format: - -```yaml -# snowddl/tech_role.yaml -DBT_STRIPE_ROLE: - comment: dbt service role for Stripe transformations - grants: - # SCHEMA grants - the key to eliminating drift! - SCHEMA:USAGE: - - SOURCE_STRIPE.STRIPE_WHY - - PROJ_STRIPE.PROJ_STRIPE - SCHEMA:CREATE TABLE,CREATE VIEW,CREATE PROCEDURE,MODIFY: - - PROJ_STRIPE.PROJ_STRIPE - # DATABASE grants - DATABASE:USAGE,CREATE SCHEMA: - - SOURCE_STRIPE - - PROJ_STRIPE - # WAREHOUSE grants - WAREHOUSE:USAGE: - - TRANSFORMING - future_grants: - TABLE:SELECT: - - SOURCE_STRIPE - TABLE:SELECT,INSERT,UPDATE,DELETE,TRUNCATE: - - PROJ_STRIPE -``` - -**Grant format**: `::` followed by list of fully-qualified object names. - -### Understanding Plan Drift (REVOKE Statements) - -When `uv run snowddl-plan` shows many REVOKE statements: - -``` -REVOKE USAGE ON SCHEMA "SOURCE_STRIPE"."STRIPE_WHY" FROM ROLE "BI_WRITER_TECH_ROLE__T_ROLE"; -REVOKE ALL ON SCHEMA "PROJ_STRIPE"."PROJ_STRIPE" FROM ROLE "DBT_STRIPE_ROLE__T_ROLE"; -``` - -This means: -1. Grants exist in Snowflake that aren't defined in YAML -2. SnowDDL wants to revoke them to match the declared state -3. These often come from dbt or manual GRANT commands - -**Common drift sources**: -- dbt runs that create schemas and grant permissions -- Manual `GRANT` commands run by admins -- Other tools (Lightdash, Omni) that manage their own permissions - -**Solution**: Add the missing `SCHEMA:` grants to `tech_role.yaml`. - -### Eliminating Schema Drift - -**Problem**: Every plan shows 100s of REVOKE statements for schema grants. - -**Root cause**: dbt and other tools grant schema permissions that aren't tracked in SnowDDL. - -**Solution steps**: -1. Run `uv run snowddl-plan` and capture output -2. Parse REVOKE statements to identify: - - Which roles need grants - - Which schemas they need access to - - What privileges they have -3. Add `SCHEMA:` entries to appropriate roles in `tech_role.yaml` -4. Re-run plan to verify drift is eliminated - -**Example parsing**: -``` -REVOKE USAGE ON SCHEMA "SOURCE_STRIPE"."STRIPE_WHY" FROM ROLE "BI_WRITER_TECH_ROLE__T_ROLE" - โ†“ โ†“ โ†“ - Privilege Schema name Role name (without __T_ROLE suffix) -``` - -Add to `tech_role.yaml`: -```yaml -BI_WRITER_TECH_ROLE: - grants: - SCHEMA:USAGE: - - SOURCE_STRIPE.STRIPE_WHY -``` - -### SnowDDL Object Type Reference - -| Object Type | Grant Key | Example | -|------------|-----------|---------| -| Database | `DATABASE:` | `DATABASE:USAGE,CREATE SCHEMA` | -| Schema | `SCHEMA:` | `SCHEMA:USAGE,CREATE TABLE` | -| Table | `TABLE:` | `TABLE:SELECT,INSERT` | -| View | `VIEW:` | `VIEW:SELECT` | -| Warehouse | `WAREHOUSE:` | `WAREHOUSE:USAGE,OPERATE` | -| Stage | `STAGE:` | `STAGE:USAGE,READ,WRITE` | -| Function | `FUNCTION:` | `FUNCTION:USAGE` | -| Procedure | `PROCEDURE:` | `PROCEDURE:USAGE` | - -### SnowDDL Documentation Links - -- [Schema Config](https://docs.snowddl.com/basic/yaml-configs/schema) -- [Technical Roles](https://docs.snowddl.com/basic/yaml-configs/technical-role) -- [Business Roles](https://docs.snowddl.com/basic/yaml-configs/business-role) -- [Permission Model](https://docs.snowddl.com/basic/yaml-configs/permission-model) -- [Env Prefix Guide](https://docs.snowddl.com/guides/other-guides/env-prefix) - -## Getting Help - -- **Troubleshooting**: `docs/guide/TROUBLESHOOTING.md` -- **CLI Reference**: `docs/guide/MANAGEMENT_COMMANDS.md` -- **Architecture**: `docs/guide/ARCHITECTURE.md` diff --git a/docs/llm-context/CONTEXT.md b/docs/llm-context/CONTEXT.md deleted file mode 100644 index 53d6140..0000000 --- a/docs/llm-context/CONTEXT.md +++ /dev/null @@ -1,90 +0,0 @@ -# Project Context - -## Domain Knowledge - -### What is SnowDDL? -SnowDDL is an open-source tool for managing Snowflake infrastructure as code. It: -- Reads YAML configuration files -- Compares desired state with actual Snowflake state -- Generates and executes DDL statements to reconcile differences - -### What is SnowTower? -SnowTower is our enterprise wrapper around SnowDDL that adds: -- User lifecycle management with MFA compliance -- Cost optimization and monitoring -- Security policies and network controls -- CLI commands via UV package manager - -## Key Concepts - -### Infrastructure as Code (IaC) -All Snowflake objects are defined in YAML files under `snowddl/`: -- Changes are version-controlled in git -- Deployments go through CI/CD (GitHub Actions) -- `plan` shows changes, `apply` executes them - -### Role-Based Access Control (RBAC) -``` -ACCOUNTADMIN (top-level) - โ†“ -SYSADMIN โ†’ USERADMIN โ†’ SECURITYADMIN - โ†“ โ†“ โ†“ -Business Roles (__B_ROLE) - โ†“ -Technical Roles (__T_ROLE) - โ†“ -Object Permissions -``` - -### User Types -- **PERSON**: Human users, require MFA, have network policies -- **SERVICE**: Service accounts, use RSA keys, typically unrestricted - -### Authentication Methods -1. **RSA Key Pairs** - Passwordless, most secure -2. **Encrypted Passwords** - Fernet encryption, fallback only -3. **MFA** - Multi-factor auth, mandatory for humans by 2026 - -## Technology Stack - -| Component | Technology | -|-----------|------------| -| Infrastructure | Snowflake (cloud data warehouse) | -| IaC Tool | SnowDDL | -| Language | Python 3.10+ | -| Package Manager | UV (fast pip alternative) | -| CI/CD | GitHub Actions | -| Encryption | Fernet (symmetric) | - -## Snowflake-Specific Knowledge - -### Object Hierarchy -``` -Account - โ””โ”€โ”€ Database - โ””โ”€โ”€ Schema - โ”œโ”€โ”€ Tables - โ”œโ”€โ”€ Views - โ”œโ”€โ”€ Stages - โ””โ”€โ”€ Procedures -``` - -### Common Privileges -- `USAGE` - Access to use an object -- `SELECT` - Read data -- `INSERT/UPDATE/DELETE` - Modify data -- `CREATE TABLE/VIEW` - Create objects -- `OWNERSHIP` - Full control - -### SnowDDL Limitations -- Does not manage SCHEMA objects (conflicts with dbt) -- Schema grants must be handled separately -- Some object types require `--apply-unsafe` flag - -## Project History - -- **2024**: Initial SnowDDL implementation -- **2025 Q1**: Security hardening (MFA, network policies) -- **2025 Q2**: CLI consolidation, UV migration -- **2025 Q3**: Repository unification (snowtower-cli merged) -- **Current**: Documentation restructure, agent support diff --git a/docs/llm-context/PATTERNS.md b/docs/llm-context/PATTERNS.md deleted file mode 100644 index d260219..0000000 --- a/docs/llm-context/PATTERNS.md +++ /dev/null @@ -1,491 +0,0 @@ -# Code Patterns and Conventions - -## Python Patterns - -### Script Template -Every script in `scripts/` should follow this pattern: - -```python -#!/usr/bin/env python3 -""" -Script description. - -Usage: - uv run command-name [options] -""" - -from dotenv import load_dotenv -load_dotenv() # ALWAYS FIRST - -import argparse -import sys -from pathlib import Path - -# Add src to path if needed -sys.path.insert(0, str(Path(__file__).parent.parent)) - -from src.snowddl_core import SnowDDLClient - - -def parse_args(): - parser = argparse.ArgumentParser( - description="Command description", - formatter_class=argparse.RawDescriptionHelpFormatter - ) - parser.add_argument( - "--verbose", "-v", - action="store_true", - help="Enable verbose output" - ) - parser.add_argument( - "--dry-run", - action="store_true", - help="Show what would be done without executing" - ) - return parser.parse_args() - - -def main(): - args = parse_args() - - try: - client = SnowDDLClient() - # Implementation here - - except Exception as e: - print(f"Error: {e}", file=sys.stderr) - sys.exit(1) - - -if __name__ == "__main__": - main() -``` - -### CLI Wrapper Pattern -In `src/management_cli.py`: - -```python -def my_command(): - """Short description for --help.""" - from scripts.my_script import main - main() -``` - -### Configuration Loading Pattern -```python -import os -from dotenv import load_dotenv - -load_dotenv() - -SNOWFLAKE_CONFIG = { - "account": os.getenv("SNOWFLAKE_ACCOUNT"), - "user": os.getenv("SNOWFLAKE_USER"), - "password": os.getenv("SNOWFLAKE_PASSWORD"), - "role": os.getenv("SNOWFLAKE_ROLE", "SYSADMIN"), - "warehouse": os.getenv("SNOWFLAKE_WAREHOUSE", "COMPUTE_WH"), -} -``` - -## YAML Patterns - -### User Definition -```yaml -# Standard human user -USERNAME: - type: PERSON - default_role: USER_ROLE - default_warehouse: USER_WH - must_change_password: false - network_policy: OFFICE_NETWORK_POLICY - comment: "Description of user purpose" -``` - -### Service Account with RSA -```yaml -# Service account (no password) -SERVICE_NAME: - type: SERVICE - default_role: SERVICE_ROLE - default_warehouse: SERVICE_WH - rsa_public_key: | - -----BEGIN PUBLIC KEY----- - MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A... - -----END PUBLIC KEY----- - comment: "Service account for X integration" -``` - -### Role with Grants -```yaml -ROLE_NAME: - comment: "Role description" - grants: - - database: DATABASE_NAME - privileges: [USAGE] - - schema: DATABASE_NAME.SCHEMA_NAME - privileges: [USAGE, SELECT] - - warehouse: WAREHOUSE_NAME - privileges: [USAGE] -``` - -### Network Policy -```yaml -POLICY_NAME: - allowed_ip_list: - - "10.0.0.0/8" - - "192.168.1.0/24" - blocked_ip_list: [] - comment: "Office network access only" -``` - -## Naming Conventions - -### Files -- Scripts: `snake_case.py` (e.g., `manage_users.py`) -- YAML configs: `snake_case.yaml` (e.g., `network_policy.yaml`) -- Documentation: `UPPER_CASE.md` (e.g., `QUICKSTART.md`) - -### Snowflake Objects -- Users: `UPPERCASE` (e.g., `ANALYST_USER`) -- Roles: `UPPERCASE_ROLE` (e.g., `ANALYST_ROLE`) -- Warehouses: `UPPERCASE_WH` (e.g., `COMPUTE_WH`) -- Databases: `UPPERCASE` (e.g., `RAW`, `ANALYTICS`) - -### Python -- Functions: `snake_case` -- Classes: `PascalCase` -- Constants: `UPPER_SNAKE_CASE` -- Private: `_leading_underscore` - -## Error Handling Pattern - -```python -import sys -from typing import Optional - - -class SnowTowerError(Exception): - """Base exception for SnowTower errors.""" - pass - - -class ConfigurationError(SnowTowerError): - """Configuration-related errors.""" - pass - - -def safe_operation(func): - """Decorator for safe operation execution.""" - def wrapper(*args, **kwargs): - try: - return func(*args, **kwargs) - except SnowTowerError as e: - print(f"Error: {e}", file=sys.stderr) - sys.exit(1) - except Exception as e: - print(f"Unexpected error: {e}", file=sys.stderr) - sys.exit(2) - return wrapper -``` - -## Testing Patterns - -### Unit Test -```python -import pytest -from scripts.my_script import my_function - - -def test_my_function_success(): - result = my_function("valid_input") - assert result == expected_output - - -def test_my_function_error(): - with pytest.raises(ValueError): - my_function("invalid_input") -``` - -### Integration Test -```python -import pytest -from unittest.mock import patch - - -@pytest.fixture -def mock_snowflake(): - with patch("src.snowddl_core.SnowDDLClient") as mock: - yield mock - - -def test_integration(mock_snowflake): - mock_snowflake.return_value.execute.return_value = [] - # Test implementation -``` - -## SnowDDL Role Hierarchy Patterns - -### Role Types and Naming - -SnowDDL automatically creates roles with specific suffixes: - -| Role Type | Suffix | Purpose | Example | -|-----------|--------|---------|---------| -| User Role | `__U_ROLE` | Per-user role | `DLT__U_ROLE` | -| Technical Role | `__T_ROLE` | Service/app permissions | `DLT_STRIPE_TECH_ROLE__T_ROLE` | -| Business Role | `__B_ROLE` | Logical groupings | `DLT_STRIPE_ROLE__B_ROLE` | -| Schema Owner | `__S_ROLE` | Object ownership | `SOURCE_STRIPE__STRIPE_WHY__OWNER__S_ROLE` | -| Database Owner | `__D_ROLE` | Database-level ownership | `SOURCE_STRIPE__OWNER__D_ROLE` | - -### Role Inheritance Hierarchy - -``` -User Role (DLT__U_ROLE) - โ”œโ”€โ”€ Business Role (DLT_STRIPE_ROLE__B_ROLE) - โ”‚ โ”œโ”€โ”€ Technical Role (DLT_STRIPE_TECH_ROLE__T_ROLE) - โ”‚ โ””โ”€โ”€ Schema Owner Role (SOURCE_STRIPE__STRIPE_WHY__OWNER__S_ROLE) - โ”‚ - โ””โ”€โ”€ Technical Role (directly granted if needed) -``` - -**Key insight**: Business roles INHERIT from technical roles AND can have schema_owner. -This means permissions flow: User โ†’ Business โ†’ Technical + Schema Owner. - -## Data Loading (DLT) Permission Patterns - -### The Table Stage Problem - -When DLT loads data, it uses Snowflake's `COPY INTO` with table stages: - -```sql -COPY INTO my_table FROM @my_table -- @table_name is the table stage -``` - -**Critical requirement**: Only the TABLE OWNER can access table stages. - -If DLT doesn't own the tables (common with SnowDDL's default SCHEMA_OWNER permission model), -loading fails with: - -``` -SQL access control error: Insufficient privileges to operate on table stage 'CHARGE' -``` - -### Solution: Grant Schema Owner Role to Business Role - -SnowDDL's `schema_owner` property on business roles is the correct solution: - -```yaml -# snowddl/business_role.yaml -DLT_STRIPE_ROLE: - comment: DLT Stripe pipeline - includes schema owner for table stage access - tech_roles: - - DLT_STRIPE_TECH_ROLE - schema_owner: - - SOURCE_STRIPE.STRIPE_WHY # Grants SOURCE_STRIPE__STRIPE_WHY__OWNER__S_ROLE - warehouse_usage: - - DLT -``` - -**Why this works**: -1. `schema_owner` grants the auto-created `SOURCE_STRIPE__STRIPE_WHY__OWNER__S_ROLE` to the business role -2. The user's role inherits from the business role (DLT__U_ROLE โ†’ DLT_STRIPE_ROLE__B_ROLE) -3. Therefore, the DLT user can access table stages through this inheritance chain - -### What Doesn't Work (and Why) - -**Technical roles don't support `schema_owner`:** -```yaml -# tech_role.yaml - This WON'T work! -DLT_STRIPE_TECH_ROLE: - schema_owner: # NOT SUPPORTED in tech_role.yaml - - SOURCE_STRIPE.STRIPE_WHY -``` - -SnowDDL's schema only allows: `grants`, `future_grants`, `account_grants`, `comment` - -**Role grants in technical roles don't work:** -```yaml -# tech_role.yaml - This ALSO won't work! -DLT_STRIPE_TECH_ROLE: - grants: - ROLE: # NOT a valid object type for grants - - SOURCE_STRIPE__STRIPE_WHY__OWNER__S_ROLE -``` - -Grant keys must be `OBJECT_TYPE:PRIVILEGE`, not just `ROLE`. - -### Complete DLT Permission Configuration - -**Technical Role** (`tech_role.yaml`): -```yaml -DLT_STRIPE_TECH_ROLE: - comment: DLT Stripe pipeline permissions - future_grants: - FILE_FORMAT:USAGE: - - SOURCE_STRIPE - SEQUENCE:USAGE: - - SOURCE_STRIPE - STAGE:USAGE,READ,WRITE: - - SOURCE_STRIPE - TABLE:SELECT,INSERT,UPDATE,DELETE,TRUNCATE,REFERENCES: - - SOURCE_STRIPE - VIEW:SELECT: - - SOURCE_STRIPE - grants: - DATABASE:USAGE,CREATE SCHEMA,MONITOR,MODIFY: - - SOURCE_STRIPE - SCHEMA:USAGE,MODIFY,MONITOR,CREATE TABLE,CREATE VIEW,CREATE FILE FORMAT,CREATE STAGE,CREATE SEQUENCE: - - SOURCE_STRIPE.STRIPE_WHY - WAREHOUSE:USAGE,MONITOR,OPERATE: - - DLT -``` - -**Business Role** (`business_role.yaml`): -```yaml -DLT_STRIPE_ROLE: - comment: DLT Stripe pipeline with schema owner access for table stages - tech_roles: - - DLT_STRIPE_TECH_ROLE - schema_owner: - - SOURCE_STRIPE.STRIPE_WHY # CRITICAL for table stage access - warehouse_usage: - - DLT -``` - -### Verifying the Configuration - -**Check role grants:** -```bash -snow sql --query "SHOW GRANTS OF ROLE SOURCE_STRIPE__STRIPE_WHY__OWNER__S_ROLE" -# Should show: granted to DLT_STRIPE_ROLE__B_ROLE -``` - -**Check table ownership:** -```bash -snow sql --query "SELECT TABLE_NAME, TABLE_OWNER FROM SOURCE_STRIPE.INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA = 'STRIPE_WHY'" -# Should show: TABLE_OWNER = SOURCE_STRIPE__STRIPE_WHY__OWNER__S_ROLE -``` - -**Check user inheritance:** -```bash -snow sql --query "SHOW GRANTS TO USER DLT" -# Should include: DLT_STRIPE_TECH_ROLE__T_ROLE -# User role should inherit from business role which has schema_owner -``` - -## Permission Model Patterns - -### One-Time Ownership Migration for Existing Schemas - -When adding `schema_owner` to a business role for a schema that already has objects, -existing objects need a one-time ownership transfer. SnowDDL only grants future ownership, -not retroactive ownership. - -**Problem**: DBT or other tools create objects BEFORE schema_owner is configured. -These objects remain owned by the original creator (often ACCOUNTADMIN). - -**Symptom**: After adding `schema_owner` and running SnowDDL apply: -``` -SQL access control error: Insufficient privileges to operate on table 'MY_TABLE' -SQL access control error: View 'MY_VIEW' already exists, but current role has no privileges -``` - -**Solution**: One-time ownership transfer after SnowDDL creates the schema owner role: - -```sql --- Transfer table ownership -GRANT OWNERSHIP ON ALL TABLES IN SCHEMA DB_NAME.SCHEMA_NAME - TO ROLE DB_NAME__SCHEMA_NAME__OWNER__S_ROLE COPY CURRENT GRANTS; - --- Transfer view ownership -GRANT OWNERSHIP ON ALL VIEWS IN SCHEMA DB_NAME.SCHEMA_NAME - TO ROLE DB_NAME__SCHEMA_NAME__OWNER__S_ROLE COPY CURRENT GRANTS; - --- If needed: procedures, functions, etc. -GRANT OWNERSHIP ON ALL PROCEDURES IN SCHEMA DB_NAME.SCHEMA_NAME - TO ROLE DB_NAME__SCHEMA_NAME__OWNER__S_ROLE COPY CURRENT GRANTS; -``` - -**Real example** (PROJ_STRIPE.PROJ_STRIPE for DBT): -```sql -GRANT OWNERSHIP ON ALL TABLES IN SCHEMA PROJ_STRIPE.PROJ_STRIPE - TO ROLE PROJ_STRIPE__PROJ_STRIPE__OWNER__S_ROLE COPY CURRENT GRANTS; - -GRANT OWNERSHIP ON ALL VIEWS IN SCHEMA PROJ_STRIPE.PROJ_STRIPE - TO ROLE PROJ_STRIPE__PROJ_STRIPE__OWNER__S_ROLE COPY CURRENT GRANTS; -``` - -**After migration**: Future objects are automatically owned by the schema owner role -through SnowDDL's future ownership grants. - -### Default SCHEMA_OWNER Model - -The default SnowDDL permission model creates schema owner roles with future OWNERSHIP grants: - -``` -When a table is created in SOURCE_STRIPE.STRIPE_WHY: -1. Creator initially owns it -2. Future ownership grant transfers ownership to SOURCE_STRIPE__STRIPE_WHY__OWNER__S_ROLE -3. Original creator loses table stage access -``` - -**Impact on data loaders**: DLT creates tables, but loses ownership, so it can't load data. - -**Solution**: Grant the schema owner role to DLT's business role (see above). - -### Alternative: Custom Permission Model (Not Recommended) - -You CAN create a custom permission model that doesn't transfer ownership: - -```yaml -# snowddl/permission_model.yaml -dlt_loader: - ruleset: SCHEMA_OWNER - owner_future_grants: - TABLE: [SELECT, INSERT, UPDATE, DELETE, TRUNCATE, REFERENCES] # No OWNERSHIP! -``` - -**Why this isn't recommended**: -1. Mixed ownership creates complexity -2. SnowDDL prefers consistent ownership patterns -3. The schema_owner solution on business roles is cleaner - -## Separating Read vs Write Access - -### Pattern: Source Database Read-Only for Transformers - -```yaml -# tech_role.yaml -DBT_STRIPE_ROLE: - comment: dbt role - READ-ONLY access to source, WRITE to project - future_grants: - TABLE:SELECT: - - SOURCE_STRIPE # Read-only - TABLE:SELECT,INSERT,UPDATE,DELETE,TRUNCATE: - - PROJ_STRIPE # Full write - grants: - SCHEMA:USAGE: - - SOURCE_STRIPE.STRIPE_WHY # Just USAGE, no CREATE - SCHEMA:USAGE,MODIFY,CREATE TABLE,CREATE VIEW: - - PROJ_STRIPE.PROJ_STRIPE # Full write -``` - -**Key principle**: Loaders write to SOURCE_*, transformers write to PROJ_*. - -## Git Patterns - -### Commit Messages -``` -type: short description - -Longer description if needed. - -Types: feat, fix, docs, refactor, test, chore -``` - -### Branch Names -``` -feature/description -fix/issue-description -docs/what-changed -``` diff --git a/docs/llm-context/README.md b/docs/llm-context/README.md deleted file mode 100644 index 056228a..0000000 --- a/docs/llm-context/README.md +++ /dev/null @@ -1,56 +0,0 @@ -# LLM Context Configuration - -This directory contains context files for LLMs (Claude, GPT, etc.) working with the SnowTower codebase. - -## Files - -| File | Purpose | -|------|---------| -| [CLAUDE.md](CLAUDE.md) | Detailed instructions for Claude Code | -| [CONTEXT.md](CONTEXT.md) | Project context and domain knowledge | -| [PATTERNS.md](PATTERNS.md) | Code patterns and conventions to follow | - -## Quick Start - -### For Claude Code Users - -The root `CLAUDE.md` file is automatically loaded by Claude Code. It contains: -- Project overview and architecture -- Key commands and workflows -- Code patterns to follow -- Safety guidelines - -### For Other LLMs - -Copy the contents of `CLAUDE.md` into your system prompt or context window. - -## Project-Specific Instructions - -When working with this codebase, LLMs should: - -1. **Always use UV for Python** - Never use pip directly -2. **Follow the command pattern** - Scripts go in `scripts/`, wrappers in `src/management_cli.py` -3. **Use load_dotenv()** - Every script must load environment variables -4. **Check before modifying** - Run `uv run snowddl-plan` before any changes - -## Key Directories - -``` -snowtower-snowddl/ -โ”œโ”€โ”€ snowddl/ # YAML infrastructure definitions (MODIFY WITH CARE) -โ”œโ”€โ”€ src/ # Python source code -โ”‚ โ”œโ”€โ”€ snowddl_core/ # OOP framework for SnowDDL -โ”‚ โ””โ”€โ”€ management_cli.py # CLI entry points -โ”œโ”€โ”€ scripts/ # Management scripts -โ”œโ”€โ”€ docs/ # Documentation -โ””โ”€โ”€ tests/ # Test files -``` - -## Safety Guidelines - -LLMs working with this codebase should: - -- **Never run `snowddl-apply` without user confirmation** -- **Always show plan output before applying changes** -- **Never modify production credentials** -- **Always use `--dry-run` when available** From df932fbe2d202a63989d849c5f1486ee05d1ccba Mon Sep 17 00:00:00 2001 From: database-tycoon Date: Sat, 21 Feb 2026 12:20:54 -0500 Subject: [PATCH 4/4] feat: Add v0.3 features - config validation, integration tests, secrets scanning, Terraform generator - Add `validate-config` command for pre-deployment YAML validation with cross-referencing (#19) - Add 58 integration tests covering CLI commands, authentication, YAML validation, deployment workflow (#20) - Add secrets scanning with detect-secrets pre-commit hook and TruffleHog CI job (#22) - Add `generate-terraform` command to convert SnowDDL YAML to Terraform HCL (#24) - Add v0.3 branch to CI triggers All 389 tests passing. Co-Authored-By: Claude Opus 4.6 --- .github/workflows/ci.yml | 19 +- .pre-commit-config.yaml | 13 + .secrets.baseline | 233 +++++ pyproject.toml | 6 + scripts/generate_terraform.py | 753 ++++++++++++++++ scripts/validate_config.py | 813 ++++++++++++++++++ src/management_cli.py | 17 + .../test_authentication_methods.py | 208 +++++ tests/integration/test_cli_commands.py | 123 +++ tests/integration/test_deployment_workflow.py | 203 +++++ tests/integration/test_yaml_validation.py | 274 ++++++ uv.lock | 15 + 12 files changed, 2675 insertions(+), 2 deletions(-) create mode 100644 .secrets.baseline create mode 100644 scripts/generate_terraform.py create mode 100644 scripts/validate_config.py create mode 100644 tests/integration/test_authentication_methods.py create mode 100644 tests/integration/test_cli_commands.py create mode 100644 tests/integration/test_deployment_workflow.py create mode 100644 tests/integration/test_yaml_validation.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0d5b0c4..7e543f0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,11 +2,26 @@ name: CI on: pull_request: - branches: [main, v0.2] + branches: [main, v0.2, v0.3] push: - branches: [main, v0.2] + branches: [main, v0.2, v0.3] jobs: + secrets-scan: + name: Secrets Scanning + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: TruffleHog secrets scan + uses: trufflesecurity/trufflehog@main + with: + extra_args: --only-verified + lint: name: Lint & Format Check runs-on: ubuntu-latest diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 722faec..2912cc7 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -14,6 +14,19 @@ repos: - id: check-merge-conflict - id: mixed-line-ending + - repo: https://github.com/Yelp/detect-secrets + rev: v1.5.0 + hooks: + - id: detect-secrets + args: ['--baseline', '.secrets.baseline'] + exclude: > + (?x)^( + .*\.lock| + .*\.yaml\.SAFE| + tests/.*| + docs/.* + )$ + - repo: local hooks: - id: validate-schema-grants diff --git a/.secrets.baseline b/.secrets.baseline new file mode 100644 index 0000000..14bbc6e --- /dev/null +++ b/.secrets.baseline @@ -0,0 +1,233 @@ +{ + "version": "1.5.0", + "plugins_used": [ + { + "name": "ArtifactoryDetector" + }, + { + "name": "AWSKeyDetector" + }, + { + "name": "AzureStorageKeyDetector" + }, + { + "name": "Base64HighEntropyString", + "limit": 4.5 + }, + { + "name": "BasicAuthDetector" + }, + { + "name": "CloudantDetector" + }, + { + "name": "DiscordBotTokenDetector" + }, + { + "name": "GitHubTokenDetector" + }, + { + "name": "GitLabTokenDetector" + }, + { + "name": "HexHighEntropyString", + "limit": 3.0 + }, + { + "name": "IbmCloudIamDetector" + }, + { + "name": "IbmCosHmacDetector" + }, + { + "name": "IPPublicDetector" + }, + { + "name": "JwtTokenDetector" + }, + { + "name": "KeywordDetector", + "keyword_exclude": "" + }, + { + "name": "MailchimpDetector" + }, + { + "name": "NpmDetector" + }, + { + "name": "OpenAIDetector" + }, + { + "name": "PrivateKeyDetector" + }, + { + "name": "PypiTokenDetector" + }, + { + "name": "SendGridDetector" + }, + { + "name": "SlackDetector" + }, + { + "name": "SoftlayerDetector" + }, + { + "name": "SquareOAuthDetector" + }, + { + "name": "StripeDetector" + }, + { + "name": "TelegramBotTokenDetector" + }, + { + "name": "TwilioKeyDetector" + } + ], + "filters_used": [ + { + "path": "detect_secrets.filters.allowlist.is_line_allowlisted" + }, + { + "path": "detect_secrets.filters.common.is_ignored_due_to_verification_policies", + "min_level": 2 + }, + { + "path": "detect_secrets.filters.heuristic.is_indirect_reference" + }, + { + "path": "detect_secrets.filters.heuristic.is_likely_id_string" + }, + { + "path": "detect_secrets.filters.heuristic.is_lock_file" + }, + { + "path": "detect_secrets.filters.heuristic.is_not_alphanumeric_string" + }, + { + "path": "detect_secrets.filters.heuristic.is_potential_uuid" + }, + { + "path": "detect_secrets.filters.heuristic.is_prefixed_with_dollar_sign" + }, + { + "path": "detect_secrets.filters.heuristic.is_sequential_string" + }, + { + "path": "detect_secrets.filters.heuristic.is_swagger_file" + }, + { + "path": "detect_secrets.filters.heuristic.is_templated_secret" + }, + { + "path": "detect_secrets.filters.regex.should_exclude_file", + "pattern": [ + "(\\.lock|\\.yaml\\.SAFE|tests/.*|docs/.*)" + ] + } + ], + "results": { + ".github/workflows/ci.yml": [ + { + "type": "Secret Keyword", + "filename": ".github/workflows/ci.yml", + "hashed_secret": "9fb7fe1217aed442b04c0f5e43b5d5a7d3287097", + "is_verified": false, + "line_number": 57 + } + ], + ".github/workflows/release.yml": [ + { + "type": "Secret Keyword", + "filename": ".github/workflows/release.yml", + "hashed_secret": "9fb7fe1217aed442b04c0f5e43b5d5a7d3287097", + "is_verified": false, + "line_number": 36 + } + ], + "README.md": [ + { + "type": "Secret Keyword", + "filename": "README.md", + "hashed_secret": "7165f6d407dc2fd68528da63260a913e71623e86", + "is_verified": false, + "line_number": 174 + } + ], + "pyproject.toml": [ + { + "type": "Secret Keyword", + "filename": "pyproject.toml", + "hashed_secret": "932da1e6368bff95652376e245fdd068dec8024f", + "is_verified": false, + "line_number": 121 + } + ], + "src/automation/README.md": [ + { + "type": "Secret Keyword", + "filename": "src/automation/README.md", + "hashed_secret": "373ee161fc7eb63bc82d686e6a855b2417f4cdbc", + "is_verified": false, + "line_number": 300 + } + ], + "src/snowddl_core/safety/agent_implementation_plan.py": [ + { + "type": "Secret Keyword", + "filename": "src/snowddl_core/safety/agent_implementation_plan.py", + "hashed_secret": "638bac731294171648258260ff2af4a09bc02aa2", + "is_verified": false, + "line_number": 41 + } + ], + "src/snowddl_core/snowddl_types.py": [ + { + "type": "Secret Keyword", + "filename": "src/snowddl_core/snowddl_types.py", + "hashed_secret": "644a049372cae070675af379d343ec1267880fe7", + "is_verified": false, + "line_number": 64 + } + ], + "src/snowtower_core/audit.py": [ + { + "type": "Secret Keyword", + "filename": "src/snowtower_core/audit.py", + "hashed_secret": "94732424e16be827cd46aa294394ab3ce93b55f4", + "is_verified": false, + "line_number": 63 + } + ], + "src/snowtower_core/managers.py": [ + { + "type": "Secret Keyword", + "filename": "src/snowtower_core/managers.py", + "hashed_secret": "5baa61e4c9b93f3f0682250b6cf8331b7ee68fd8", + "is_verified": false, + "line_number": 164 + } + ], + "src/snowtower_core/models.py": [ + { + "type": "Secret Keyword", + "filename": "src/snowtower_core/models.py", + "hashed_secret": "112bb791304791ddcf692e29fd5cf149b35fea37", + "is_verified": false, + "line_number": 36 + } + ], + "src/user_management/health_check.py": [ + { + "type": "Secret Keyword", + "filename": "src/user_management/health_check.py", + "hashed_secret": "5baa61e4c9b93f3f0682250b6cf8331b7ee68fd8", + "is_verified": false, + "line_number": 42 + } + ] + }, + "generated_at": "2026-02-21T17:15:41Z" +} diff --git a/pyproject.toml b/pyproject.toml index 9433c7d..07eb28a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -50,6 +50,7 @@ dev = [ # API documentation generation "mkdocstrings[python]>=0.24.0", "mkdocstrings-python>=1.8.0", + "detect-secrets>=1.5.0", ] [build-system] @@ -85,6 +86,7 @@ manage-security = "src.management_cli:security" manage-backup = "src.management_cli:backup" apply-schema-grants = "src.management_cli:apply_schema_grants" validate-schema-grants = "src.management_cli:validate_schema_grants" +validate-config = "src.management_cli:validate_config" test-schema-mgmt = "scripts.test_schema_management:main" # === MONITORING & OBSERVABILITY === @@ -100,6 +102,10 @@ util-diagnose-auth = "src.diagnose_auth:main" util-fix-auth = "src.fix_auth:main" generate-rsa-batch = "scripts.generate_rsa_keys_batch:main" +# === TERRAFORM GENERATION === +# Convert SnowDDL YAML to Terraform HCL +generate-terraform = "src.management_cli:generate_terraform" + # === AUTOMATION === # GitHub issue to SnowDDL automation with PR creation github-to-snowddl = "src.management_cli:github_to_snowddl" diff --git a/scripts/generate_terraform.py b/scripts/generate_terraform.py new file mode 100644 index 0000000..ff9528c --- /dev/null +++ b/scripts/generate_terraform.py @@ -0,0 +1,753 @@ +from dotenv import load_dotenv + +load_dotenv() + +""" +Generate Terraform HCL files from SnowDDL YAML configurations. + +Converts SnowDDL YAML infrastructure definitions into Terraform-compatible +HCL or JSON files for the Snowflake-Labs/snowflake provider. + +Usage: + uv run generate-terraform # Print to stdout + uv run generate-terraform --output ./terraform/ # Write separate .tf files + uv run generate-terraform --format json # JSON output +""" + +import argparse +import json +import re +import sys +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Dict, List, Optional, Tuple + +import yaml + + +# --------------------------------------------------------------------------- +# Custom YAML loader that handles the !decrypt tag gracefully +# --------------------------------------------------------------------------- + +class SafeLoaderWithDecrypt(yaml.SafeLoader): + """YAML loader that treats !decrypt tagged values as placeholder strings.""" + pass + + +def _decrypt_constructor(loader: yaml.SafeLoader, node: yaml.ScalarNode) -> str: + """Return a placeholder for encrypted values.""" + return "" + + +SafeLoaderWithDecrypt.add_constructor("!decrypt", _decrypt_constructor) + + +# --------------------------------------------------------------------------- +# Constants +# --------------------------------------------------------------------------- + +WAREHOUSE_SIZE_MAP = { + "X-Small": "XSMALL", + "x-small": "XSMALL", + "xsmall": "XSMALL", + "Small": "SMALL", + "small": "SMALL", + "Medium": "MEDIUM", + "medium": "MEDIUM", + "Large": "LARGE", + "large": "LARGE", + "X-Large": "XLARGE", + "x-large": "XLARGE", + "xlarge": "XLARGE", + "2X-Large": "XXLARGE", + "2x-large": "XXLARGE", + "3X-Large": "XXXLARGE", + "3x-large": "XXXLARGE", + "4X-Large": "X4LARGE", + "4x-large": "X4LARGE", +} + +HEADER = """\ +# Generated by SnowTower - DO NOT EDIT MANUALLY +# Source: snowddl/ YAML configurations +""" + +PROVIDER_BLOCK = """\ +terraform { + required_providers { + snowflake = { + source = "Snowflake-Labs/snowflake" + version = "~> 1.0" + } + } +} +""" + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +def to_tf_name(name: str) -> str: + """Convert a Snowflake object name to a valid Terraform resource name. + + Rules: + - Lowercase + - Replace non-alphanumeric characters with underscores + - Strip leading/trailing underscores + - Collapse consecutive underscores + """ + result = name.lower() + result = re.sub(r"[^a-z0-9]", "_", result) + result = re.sub(r"_+", "_", result) + result = result.strip("_") + return result + + +def hcl_value(value: Any) -> str: + """Format a Python value as an HCL literal.""" + if isinstance(value, bool): + return "true" if value else "false" + if isinstance(value, int): + return str(value) + if isinstance(value, float): + return str(value) + if isinstance(value, str): + # Escape backslashes and quotes inside the string + escaped = value.replace("\\", "\\\\").replace('"', '\\"') + return f'"{escaped}"' + if isinstance(value, list): + items = ", ".join(hcl_value(v) for v in value) + return f"[{items}]" + return f'"{value}"' + + +def hcl_block(resource_type: str, resource_name: str, attrs: Dict[str, Any], + import_id: Optional[str] = None) -> str: + """Render a single Terraform resource block as HCL. + + Parameters + ---------- + resource_type : str + e.g. ``snowflake_user`` + resource_name : str + The Terraform resource name (snake_case) + attrs : dict + Key/value pairs for the resource body + import_id : str, optional + If provided an ``import {}`` block is emitted before the resource. + """ + lines: List[str] = [] + + if import_id is not None: + lines.append(f'import {{') + lines.append(f' to = {resource_type}.{resource_name}') + lines.append(f' id = "{import_id}"') + lines.append(f'}}') + lines.append("") + + lines.append(f'resource "{resource_type}" "{resource_name}" {{') + max_key_len = max((len(k) for k in attrs), default=0) + + for key, value in attrs.items(): + if isinstance(value, dict): + # Nested block (e.g. triggers in resource monitors) + lines.append(f" {key} {{") + for nk, nv in value.items(): + lines.append(f" {nk:{max_key_len}} = {hcl_value(nv)}") + lines.append(" }") + else: + lines.append(f" {key:{max_key_len}} = {hcl_value(value)}") + + lines.append("}") + return "\n".join(lines) + + +def load_yaml(path: Path) -> Optional[Dict]: + """Load a YAML file, returning None if missing or empty.""" + if not path.exists(): + return None + with open(path) as f: + data = yaml.load(f, Loader=SafeLoaderWithDecrypt) + return data if isinstance(data, dict) else None + + +# --------------------------------------------------------------------------- +# Generators -- each returns a list of HCL strings +# --------------------------------------------------------------------------- + +def generate_users(snowddl_dir: Path) -> List[str]: + """Generate snowflake_user resources from user.yaml.""" + data = load_yaml(snowddl_dir / "user.yaml") + if not data: + return [] + + blocks: List[str] = [] + for name, cfg in sorted(data.items()): + attrs: Dict[str, Any] = {"name": name} + for yaml_key, tf_key in [ + ("email", "email"), + ("first_name", "first_name"), + ("last_name", "last_name"), + ("default_role", "default_role"), + ("comment", "comment"), + ("type", "user_type"), + ]: + if yaml_key in cfg: + attrs[tf_key] = cfg[yaml_key] + + if "rsa_public_key" in cfg: + raw_key = cfg["rsa_public_key"].strip() + attrs["rsa_public_key"] = raw_key + + tf_name = to_tf_name(name) + blocks.append(hcl_block("snowflake_user", tf_name, attrs, import_id=name)) + + return blocks + + +def generate_warehouses(snowddl_dir: Path) -> List[str]: + """Generate snowflake_warehouse resources from warehouse.yaml.""" + data = load_yaml(snowddl_dir / "warehouse.yaml") + if not data: + return [] + + blocks: List[str] = [] + for name, cfg in sorted(data.items()): + size_raw = cfg.get("size", "X-Small") + size_tf = WAREHOUSE_SIZE_MAP.get(size_raw, size_raw.upper().replace("-", "")) + + attrs: Dict[str, Any] = { + "name": name, + "warehouse_size": size_tf, + } + + if "auto_suspend" in cfg: + attrs["auto_suspend"] = cfg["auto_suspend"] + attrs["auto_resume"] = cfg.get("auto_resume", True) + + for yaml_key, tf_key in [ + ("comment", "comment"), + ("resource_monitor", "resource_monitor"), + ("min_cluster_count", "min_cluster_count"), + ("max_cluster_count", "max_cluster_count"), + ]: + if yaml_key in cfg: + attrs[tf_key] = cfg[yaml_key] + + tf_name = to_tf_name(name) + blocks.append(hcl_block("snowflake_warehouse", tf_name, attrs, import_id=name)) + + return blocks + + +def generate_business_roles(snowddl_dir: Path) -> Tuple[List[str], List[str]]: + """Generate snowflake_account_role and grant resources from business_role.yaml. + + Returns (role_blocks, grant_blocks). + """ + data = load_yaml(snowddl_dir / "business_role.yaml") + if not data: + return [], [] + + role_blocks: List[str] = [] + grant_blocks: List[str] = [] + + for name, cfg in sorted(data.items()): + sf_role_name = f"{name}__B_ROLE" + tf_name = to_tf_name(sf_role_name) + + attrs = { + "name": sf_role_name, + } + if "comment" in cfg: + attrs["comment"] = cfg["comment"] + + role_blocks.append(hcl_block("snowflake_account_role", tf_name, attrs, + import_id=sf_role_name)) + + # Tech role inheritance grants + for tech_role in cfg.get("tech_roles", []): + tech_sf_name = f"{tech_role}__T_ROLE" + grant_tf_name = to_tf_name(f"{sf_role_name}_inherits_{tech_sf_name}") + grant_attrs = { + "role_name": tech_sf_name, + "parent_role_name": sf_role_name, + } + grant_blocks.append(hcl_block( + "snowflake_grant_account_role", grant_tf_name, grant_attrs)) + + # Warehouse usage grants + for wh in cfg.get("warehouse_usage", []): + grant_tf_name = to_tf_name(f"{sf_role_name}_wh_{wh}") + grant_attrs = { + "account_role_name": sf_role_name, + "privileges": ["USAGE"], + "on_account_object": {"object_type": "WAREHOUSE", "object_name": wh}, + } + grant_blocks.append(hcl_block( + "snowflake_grant_privileges_to_account_role", grant_tf_name, grant_attrs)) + + # Schema owner grants + for schema_ref in cfg.get("schema_owner", []): + parts = schema_ref.split(".") + if len(parts) == 2: + db, schema = parts + owner_role = f"{db}__{schema}__OWNER__S_ROLE" + grant_tf_name = to_tf_name(f"{sf_role_name}_owner_{db}_{schema}") + grant_attrs = { + "role_name": owner_role, + "parent_role_name": sf_role_name, + } + grant_blocks.append(hcl_block( + "snowflake_grant_account_role", grant_tf_name, grant_attrs)) + + return role_blocks, grant_blocks + + +def _parse_grant_key(grant_key: str) -> Tuple[str, List[str]]: + """Parse a SnowDDL grant key like ``DATABASE:USAGE,CREATE SCHEMA``. + + Returns (object_type, privilege_list). + """ + parts = grant_key.split(":", 1) + obj_type = parts[0] + privileges = [p.strip() for p in parts[1].split(",")] + return obj_type, privileges + + +def generate_tech_roles(snowddl_dir: Path) -> Tuple[List[str], List[str]]: + """Generate snowflake_account_role and grant resources from tech_role.yaml. + + Returns (role_blocks, grant_blocks). + """ + data = load_yaml(snowddl_dir / "tech_role.yaml") + if not data: + return [], [] + + role_blocks: List[str] = [] + grant_blocks: List[str] = [] + + for name, cfg in sorted(data.items()): + sf_role_name = f"{name}__T_ROLE" + tf_name = to_tf_name(sf_role_name) + + attrs = {"name": sf_role_name} + if "comment" in cfg: + attrs["comment"] = cfg["comment"] + + role_blocks.append(hcl_block("snowflake_account_role", tf_name, attrs, + import_id=sf_role_name)) + + # Regular grants + for grant_key, targets in (cfg.get("grants") or {}).items(): + obj_type, privileges = _parse_grant_key(grant_key) + if not targets: + continue + for target in targets: + grant_tf_name = to_tf_name( + f"{sf_role_name}_grant_{obj_type}_{target}_{'_'.join(privileges)}" + ) + grant_attrs: Dict[str, Any] = { + "account_role_name": sf_role_name, + "privileges": privileges, + } + + if obj_type == "SCHEMA": + # target is DATABASE.SCHEMA + grant_attrs["on_schema"] = { + "schema_name": f'"{target}"', + } + elif obj_type in ("DATABASE", "WAREHOUSE"): + grant_attrs["on_account_object"] = { + "object_type": obj_type, + "object_name": target, + } + else: + grant_attrs["on_account_object"] = { + "object_type": obj_type, + "object_name": target, + } + + grant_blocks.append(hcl_block( + "snowflake_grant_privileges_to_account_role", + grant_tf_name, grant_attrs)) + + # Future grants + for grant_key, targets in (cfg.get("future_grants") or {}).items(): + obj_type, privileges = _parse_grant_key(grant_key) + if not targets: + continue + for target in targets: + grant_tf_name = to_tf_name( + f"{sf_role_name}_future_{obj_type}_{target}_{'_'.join(privileges)}" + ) + grant_attrs = { + "account_role_name": sf_role_name, + "privileges": privileges, + "on_schema_object": { + "object_type_plural": f"{obj_type}S", + "in_database": target, + }, + "all_privileges": False, + "with_grant_option": False, + } + + grant_blocks.append(hcl_block( + "snowflake_grant_privileges_to_account_role", + grant_tf_name, grant_attrs)) + + return role_blocks, grant_blocks + + +def generate_network_policies(snowddl_dir: Path) -> List[str]: + """Generate snowflake_network_policy resources from network_policy.yaml.""" + data = load_yaml(snowddl_dir / "network_policy.yaml") + if not data: + return [] + + blocks: List[str] = [] + for name, cfg in sorted(data.items()): + sf_name = name.upper() + attrs: Dict[str, Any] = {"name": sf_name} + + if "allowed_ip_list" in cfg: + attrs["allowed_ip_list"] = cfg["allowed_ip_list"] + if "comment" in cfg: + attrs["comment"] = cfg["comment"] + + tf_name = to_tf_name(name) + blocks.append(hcl_block("snowflake_network_policy", tf_name, attrs, + import_id=sf_name)) + + return blocks + + +def generate_authentication_policies(snowddl_dir: Path) -> List[str]: + """Generate snowflake_authentication_policy resources.""" + data = load_yaml(snowddl_dir / "authentication_policy.yaml") + if not data: + return [] + + blocks: List[str] = [] + for name, cfg in sorted(data.items()): + sf_name = name.upper() + attrs: Dict[str, Any] = {"name": sf_name} + + for yaml_key, tf_key in [ + ("comment", "comment"), + ("client_types", "client_types"), + ("mfa_authentication_methods", "mfa_authentication_methods"), + ("mfa_enrollment", "mfa_enrollment"), + ("authentication_methods", "authentication_methods"), + ("security_integrations", "security_integrations"), + ]: + if yaml_key in cfg: + attrs[tf_key] = cfg[yaml_key] + + tf_name = to_tf_name(name) + blocks.append(hcl_block("snowflake_authentication_policy", tf_name, attrs, + import_id=sf_name)) + + return blocks + + +def generate_password_policies(snowddl_dir: Path) -> List[str]: + """Generate snowflake_password_policy resources.""" + data = load_yaml(snowddl_dir / "password_policy.yaml") + if not data: + return [] + + blocks: List[str] = [] + for name, cfg in sorted(data.items()): + sf_name = name.upper() + attrs: Dict[str, Any] = {"name": sf_name} + + # Map all password_* fields and other known fields + for key, value in sorted(cfg.items()): + if key == "comment_in_sql": + continue # SnowDDL-specific, not relevant for Terraform + attrs[key] = value + + tf_name = to_tf_name(name) + blocks.append(hcl_block("snowflake_password_policy", tf_name, attrs, + import_id=sf_name)) + + return blocks + + +def generate_session_policies(snowddl_dir: Path) -> List[str]: + """Generate snowflake_session_policy resources.""" + data = load_yaml(snowddl_dir / "session_policy.yaml") + if not data: + return [] + + blocks: List[str] = [] + for name, cfg in sorted(data.items()): + sf_name = name.upper() + attrs: Dict[str, Any] = {"name": sf_name} + + for key, value in sorted(cfg.items()): + if key == "comment_in_sql": + continue + attrs[key] = value + + tf_name = to_tf_name(name) + blocks.append(hcl_block("snowflake_session_policy", tf_name, attrs, + import_id=sf_name)) + + return blocks + + +def generate_resource_monitors(snowddl_dir: Path) -> List[str]: + """Generate snowflake_resource_monitor resources from resource_monitor.yaml.""" + data = load_yaml(snowddl_dir / "resource_monitor.yaml") + if not data: + return [] + + blocks: List[str] = [] + for name, cfg in sorted(data.items()): + attrs: Dict[str, Any] = {"name": name} + + if "credit_quota" in cfg: + attrs["credit_quota"] = cfg["credit_quota"] + if "frequency" in cfg: + attrs["frequency"] = cfg["frequency"] + + # Parse triggers into separate lists + triggers = cfg.get("triggers", {}) + notify_triggers: List[int] = [] + suspend_triggers: List[int] = [] + suspend_immediate_triggers: List[int] = [] + + for threshold, action in sorted(triggers.items()): + action_upper = str(action).upper() + if action_upper == "NOTIFY": + notify_triggers.append(int(threshold)) + elif action_upper == "SUSPEND": + suspend_triggers.append(int(threshold)) + elif action_upper == "SUSPEND_IMMEDIATE": + suspend_immediate_triggers.append(int(threshold)) + + if notify_triggers: + attrs["notify_triggers"] = notify_triggers + if suspend_triggers: + attrs["suspend_triggers"] = suspend_triggers + if suspend_immediate_triggers: + attrs["suspend_immediate_triggers"] = suspend_immediate_triggers + + tf_name = to_tf_name(name) + blocks.append(hcl_block("snowflake_resource_monitor", tf_name, attrs, + import_id=name)) + + return blocks + + +def generate_databases(snowddl_dir: Path) -> List[str]: + """Generate snowflake_database resources from database directories.""" + blocks: List[str] = [] + + for params_path in sorted(snowddl_dir.glob("*/params.yaml")): + db_name = params_path.parent.name + cfg = load_yaml(params_path) or {} + + attrs: Dict[str, Any] = {"name": db_name} + + if "comment" in cfg: + attrs["comment"] = cfg["comment"] + if cfg.get("is_sandbox"): + attrs["comment"] = attrs.get("comment", f"Sandbox database {db_name}") + + tf_name = to_tf_name(db_name) + blocks.append(hcl_block("snowflake_database", tf_name, attrs, + import_id=db_name)) + + return blocks + + +# --------------------------------------------------------------------------- +# Output formatting +# --------------------------------------------------------------------------- + +@dataclass +class TerraformOutput: + """Holds all generated Terraform sections.""" + main: str = "" + users: List[str] = field(default_factory=list) + warehouses: List[str] = field(default_factory=list) + roles: List[str] = field(default_factory=list) + grants: List[str] = field(default_factory=list) + policies: List[str] = field(default_factory=list) + databases: List[str] = field(default_factory=list) + resource_monitors: List[str] = field(default_factory=list) + + +def generate_all(snowddl_dir: Path) -> TerraformOutput: + """Run all generators and return structured output.""" + output = TerraformOutput() + output.main = PROVIDER_BLOCK + + # Users + output.users = generate_users(snowddl_dir) + + # Warehouses + output.warehouses = generate_warehouses(snowddl_dir) + + # Business roles + b_role_blocks, b_grant_blocks = generate_business_roles(snowddl_dir) + + # Tech roles + t_role_blocks, t_grant_blocks = generate_tech_roles(snowddl_dir) + + output.roles = b_role_blocks + t_role_blocks + output.grants = b_grant_blocks + t_grant_blocks + + # Policies (network, authentication, password, session) + output.policies = ( + generate_network_policies(snowddl_dir) + + generate_authentication_policies(snowddl_dir) + + generate_password_policies(snowddl_dir) + + generate_session_policies(snowddl_dir) + ) + + # Resource monitors + output.resource_monitors = generate_resource_monitors(snowddl_dir) + + # Databases + output.databases = generate_databases(snowddl_dir) + + return output + + +def render_section(header_comment: str, blocks: List[str]) -> str: + """Combine blocks into a single string with a section header.""" + if not blocks: + return "" + lines = [HEADER, f"# {header_comment}", ""] + lines.append("\n\n".join(blocks)) + lines.append("") + return "\n".join(lines) + + +def write_to_directory(output: TerraformOutput, out_dir: Path) -> None: + """Write separate .tf files into the output directory.""" + out_dir.mkdir(parents=True, exist_ok=True) + + files = { + "main.tf": HEADER + "\n" + output.main, + "users.tf": render_section("User resources", output.users), + "warehouses.tf": render_section("Warehouse resources", output.warehouses), + "roles.tf": render_section("Role resources (business + tech)", output.roles), + "grants.tf": render_section("Grant resources", output.grants), + "policies.tf": render_section( + "Policy resources (network, authentication, password, session)", + output.policies), + "databases.tf": render_section("Database resources", output.databases), + "resource_monitors.tf": render_section( + "Resource monitor resources", output.resource_monitors), + } + + for filename, content in files.items(): + if not content.strip(): + continue + filepath = out_dir / filename + filepath.write_text(content) + print(f" Wrote {filepath}") + + +def write_to_stdout(output: TerraformOutput, fmt: str) -> None: + """Print combined output to stdout.""" + if fmt == "json": + # Collect a simple representation for JSON output + combined = { + "main": output.main, + "users": output.users, + "warehouses": output.warehouses, + "roles": output.roles, + "grants": output.grants, + "policies": output.policies, + "databases": output.databases, + "resource_monitors": output.resource_monitors, + } + print(json.dumps(combined, indent=2)) + return + + # HCL to stdout -- combine all sections + sections = [ + (HEADER + "\n" + output.main), + render_section("User resources", output.users), + render_section("Warehouse resources", output.warehouses), + render_section("Role resources (business + tech)", output.roles), + render_section("Grant resources", output.grants), + render_section("Policy resources (network, auth, password, session)", + output.policies), + render_section("Database resources", output.databases), + render_section("Resource monitor resources", output.resource_monitors), + ] + + print("\n".join(s for s in sections if s.strip())) + + +# --------------------------------------------------------------------------- +# CLI +# --------------------------------------------------------------------------- + +def main() -> None: + parser = argparse.ArgumentParser( + description="Generate Terraform HCL files from SnowDDL YAML configurations.", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog="""\ +Examples: + uv run generate-terraform # Print HCL to stdout + uv run generate-terraform --output ./terraform/ # Write separate .tf files + uv run generate-terraform --format json # JSON output to stdout +""", + ) + parser.add_argument( + "--output", "-o", + type=str, + default=None, + help="Output directory for .tf files. If omitted, prints to stdout.", + ) + parser.add_argument( + "--format", "-f", + choices=["hcl", "json"], + default="hcl", + dest="fmt", + help="Output format (default: hcl).", + ) + parser.add_argument( + "--snowddl-dir", + type=str, + default=None, + help="Path to snowddl/ directory (auto-detected from project root).", + ) + args = parser.parse_args() + + # Resolve snowddl directory + if args.snowddl_dir: + snowddl_dir = Path(args.snowddl_dir) + else: + # Walk up from this script to find snowddl/ + project_root = Path(__file__).resolve().parent.parent + snowddl_dir = project_root / "snowddl" + + if not snowddl_dir.is_dir(): + print(f"Error: snowddl directory not found at {snowddl_dir}", file=sys.stderr) + sys.exit(1) + + print(f"Reading SnowDDL configs from: {snowddl_dir}", file=sys.stderr) + + output = generate_all(snowddl_dir) + + if args.output: + out_dir = Path(args.output) + print(f"Writing Terraform files to: {out_dir}", file=sys.stderr) + write_to_directory(output, out_dir) + print(f"\nDone. Generated Terraform files in {out_dir}", file=sys.stderr) + else: + write_to_stdout(output, args.fmt) + + +if __name__ == "__main__": + main() diff --git a/scripts/validate_config.py b/scripts/validate_config.py new file mode 100644 index 0000000..c30aecb --- /dev/null +++ b/scripts/validate_config.py @@ -0,0 +1,813 @@ +#!/usr/bin/env python3 +""" +SnowDDL YAML Configuration Validator. + +Validates all SnowDDL YAML configuration files before deployment. +Performs syntax checking, field validation, and cross-reference integrity checks. + +Usage: + uv run validate-config # Validate all configs + uv run validate-config --strict # Fail on warnings too + uv run validate-config --quiet # Only show errors + uv run validate-config snowddl/user.yaml # Validate specific file(s) +""" + +from dotenv import load_dotenv + +load_dotenv() + +import argparse +import ipaddress +import re +import sys +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Dict, List, Optional, Set, Tuple + +import yaml + + +# --------------------------------------------------------------------------- +# Constants +# --------------------------------------------------------------------------- + +VALID_WAREHOUSE_SIZES: Set[str] = { + "X-Small", + "Small", + "Medium", + "Large", + "X-Large", + "2X-Large", + "3X-Large", + "4X-Large", + "5X-Large", + "6X-Large", +} + +# Also accept uppercased/alternative forms that Snowflake recognises +VALID_WAREHOUSE_SIZES_NORMALISED: Set[str] = { + s.upper().replace("-", "") for s in VALID_WAREHOUSE_SIZES +} + +VALID_GRANT_OBJECT_TYPES: Set[str] = { + "DATABASE", + "SCHEMA", + "TABLE", + "VIEW", + "WAREHOUSE", + "STAGE", + "FUNCTION", + "PROCEDURE", + "FILE_FORMAT", + "SEQUENCE", +} + +SYSTEM_ROLES: Set[str] = { + "ACCOUNTADMIN", + "SYSADMIN", + "USERADMIN", + "SECURITYADMIN", + "ORGADMIN", + "PUBLIC", +} + +VALID_USER_TYPES: Set[str] = {"PERSON", "SERVICE"} + +# SnowDDL appends __B_ROLE to business role names +BUSINESS_ROLE_SUFFIX = "__B_ROLE" + + +# --------------------------------------------------------------------------- +# Result tracking +# --------------------------------------------------------------------------- + + +@dataclass +class ValidationResult: + """Accumulates errors and warnings across all checks.""" + + errors: List[str] = field(default_factory=list) + warnings: List[str] = field(default_factory=list) + info: List[str] = field(default_factory=list) + + def error(self, message: str) -> None: + self.errors.append(message) + + def warning(self, message: str) -> None: + self.warnings.append(message) + + def ok(self, message: str) -> None: + self.info.append(message) + + @property + def has_errors(self) -> bool: + return len(self.errors) > 0 + + @property + def has_warnings(self) -> bool: + return len(self.warnings) > 0 + + +# --------------------------------------------------------------------------- +# YAML loading helpers +# --------------------------------------------------------------------------- + + +def _find_project_root() -> Path: + """Walk upwards from this script to find the project root (contains pyproject.toml).""" + current = Path(__file__).resolve().parent + for _ in range(10): + if (current / "pyproject.toml").exists(): + return current + current = current.parent + # Fallback: assume scripts/ is one level below root + return Path(__file__).resolve().parent.parent + + +def _load_yaml(path: Path) -> Tuple[Optional[Dict[str, Any]], Optional[str]]: + """Load and parse a YAML file. Returns (data, error_message).""" + if not path.exists(): + return None, f"File not found: {path}" + try: + with open(path, "r") as f: + data = yaml.safe_load(f) + if data is None: + data = {} + if not isinstance(data, dict): + return None, f"Expected a YAML mapping at top level, got {type(data).__name__}" + return data, None + except yaml.YAMLError as exc: + return None, f"YAML syntax error: {exc}" + + +# --------------------------------------------------------------------------- +# Per-file validators +# --------------------------------------------------------------------------- + + +def validate_user_yaml( + data: Dict[str, Any], + result: ValidationResult, + business_roles: Set[str], +) -> None: + """Validate user.yaml entries.""" + names_seen: Set[str] = set() + + for user_name, user_cfg in data.items(): + if not isinstance(user_cfg, dict): + result.error(f"User {user_name}: expected a mapping, got {type(user_cfg).__name__}") + continue + + # Duplicate check + upper_name = user_name.upper() + if upper_name in names_seen: + result.error(f"User {user_name}: duplicate user name") + names_seen.add(upper_name) + + # Type field + user_type = user_cfg.get("type") + if user_type is None: + result.error(f"User {user_name}: missing required 'type' field (PERSON or SERVICE)") + elif str(user_type).upper() not in VALID_USER_TYPES: + result.error( + f"User {user_name}: invalid type '{user_type}' (must be PERSON or SERVICE)" + ) + else: + user_type = str(user_type).upper() + + # PERSON-specific checks + if user_type == "PERSON": + if "email" not in user_cfg: + result.warning(f"User {user_name}: PERSON user should have 'email' field") + + # SERVICE-specific checks + if user_type == "SERVICE": + rsa_key = user_cfg.get("rsa_public_key", "") + if not rsa_key: + result.warning( + f"User {user_name}: SERVICE user should have 'rsa_public_key' field" + ) + elif "example" in str(rsa_key).lower() or "replace" in str(rsa_key).lower(): + result.warning(f"User {user_name}: RSA key appears to be a placeholder") + + # default_role cross-reference (deferred to cross-ref if business_roles provided) + default_role = user_cfg.get("default_role") + if default_role and business_roles: + role_upper = str(default_role).upper() + # Accept system roles, raw business role names, and names with __B_ROLE suffix + is_system = role_upper in SYSTEM_ROLES + is_business_raw = role_upper in business_roles + is_business_suffixed = role_upper.replace(BUSINESS_ROLE_SUFFIX, "") in business_roles + if not (is_system or is_business_raw or is_business_suffixed): + result.error( + f"User {user_name}: default_role '{default_role}' does not match any " + f"system role or business role in business_role.yaml" + ) + + count = len(data) + result.ok(f"{count} user{'s' if count != 1 else ''} validated") + + +def validate_business_role_yaml( + data: Dict[str, Any], + result: ValidationResult, + tech_roles: Set[str], + warehouses: Set[str], +) -> None: + """Validate business_role.yaml entries.""" + names_seen: Set[str] = set() + + for role_name, role_cfg in data.items(): + if not isinstance(role_cfg, dict): + result.error( + f"Business role {role_name}: expected a mapping, got {type(role_cfg).__name__}" + ) + continue + + upper_name = role_name.upper() + if upper_name in names_seen: + result.error(f"Business role {role_name}: duplicate role name") + names_seen.add(upper_name) + + # tech_roles references + for tr in role_cfg.get("tech_roles", []): + if tech_roles and str(tr).upper() not in tech_roles: + result.error( + f"Business role {role_name}: tech_role '{tr}' not found in tech_role.yaml" + ) + + # warehouse_usage references + for wh in role_cfg.get("warehouse_usage", []): + if warehouses and str(wh).upper() not in warehouses: + result.error( + f"Business role {role_name}: warehouse '{wh}' not found in warehouse.yaml" + ) + + # schema_owner format (DB.SCHEMA) + for so in role_cfg.get("schema_owner", []): + if not re.match(r"^[A-Za-z0-9_]+\.[A-Za-z0-9_]+$", str(so)): + result.error( + f"Business role {role_name}: schema_owner '{so}' is not valid " + f"DB.SCHEMA format" + ) + + count = len(data) + result.ok(f"{count} business role{'s' if count != 1 else ''} validated") + + +def validate_tech_role_yaml( + data: Dict[str, Any], + result: ValidationResult, + warehouses: Set[str], +) -> None: + """Validate tech_role.yaml entries.""" + names_seen: Set[str] = set() + + for role_name, role_cfg in data.items(): + if not isinstance(role_cfg, dict): + result.error( + f"Tech role {role_name}: expected a mapping, got {type(role_cfg).__name__}" + ) + continue + + upper_name = role_name.upper() + if upper_name in names_seen: + result.error(f"Tech role {role_name}: duplicate role name") + names_seen.add(upper_name) + + # Validate grant keys in both 'grants' and 'future_grants' + for section_name in ("grants", "future_grants"): + section = role_cfg.get(section_name, {}) + if not isinstance(section, dict): + continue + + for grant_key, targets in section.items(): + # Grant key format: OBJECT_TYPE:PRIVILEGE(S) + parts = str(grant_key).split(":", 1) + if len(parts) != 2 or not parts[1]: + result.error( + f"Tech role {role_name}: grant key '{grant_key}' in {section_name} " + f"must follow OBJECT_TYPE:PRIVILEGE format" + ) + continue + + obj_type = parts[0].upper() + if obj_type not in VALID_GRANT_OBJECT_TYPES: + result.error( + f"Tech role {role_name}: invalid object type '{parts[0]}' in " + f"grant key '{grant_key}' (valid: {', '.join(sorted(VALID_GRANT_OBJECT_TYPES))})" + ) + + # WAREHOUSE:USAGE cross-reference + if obj_type == "WAREHOUSE" and warehouses and isinstance(targets, list): + for wh in targets: + if str(wh).upper() not in warehouses: + result.error( + f"Tech role {role_name}: warehouse '{wh}' in " + f"{section_name} not found in warehouse.yaml" + ) + + count = len(data) + result.ok(f"{count} technical role{'s' if count != 1 else ''} validated") + + +def validate_warehouse_yaml( + data: Dict[str, Any], + result: ValidationResult, + resource_monitors: Set[str], +) -> None: + """Validate warehouse.yaml entries.""" + names_seen: Set[str] = set() + + for wh_name, wh_cfg in data.items(): + if not isinstance(wh_cfg, dict): + result.error( + f"Warehouse {wh_name}: expected a mapping, got {type(wh_cfg).__name__}" + ) + continue + + upper_name = wh_name.upper() + if upper_name in names_seen: + result.error(f"Warehouse {wh_name}: duplicate warehouse name") + names_seen.add(upper_name) + + # Size validation + size = wh_cfg.get("size") + if size is not None: + normalised = str(size).upper().replace("-", "") + if normalised not in VALID_WAREHOUSE_SIZES_NORMALISED: + result.error( + f"Warehouse {wh_name}: invalid size '{size}' " + f"(valid: {', '.join(sorted(VALID_WAREHOUSE_SIZES))})" + ) + + # auto_suspend validation + auto_suspend = wh_cfg.get("auto_suspend") + if auto_suspend is not None: + try: + val = int(auto_suspend) + if val < 0: + result.error( + f"Warehouse {wh_name}: auto_suspend must be a positive integer, got {val}" + ) + except (ValueError, TypeError): + result.error( + f"Warehouse {wh_name}: auto_suspend must be an integer, got '{auto_suspend}'" + ) + + # resource_monitor cross-reference + monitor = wh_cfg.get("resource_monitor") + if monitor and resource_monitors: + if str(monitor).upper() not in resource_monitors: + result.error( + f"Warehouse {wh_name}: resource_monitor '{monitor}' " + f"not found in resource_monitor.yaml" + ) + + count = len(data) + result.ok(f"{count} warehouse{'s' if count != 1 else ''} validated") + + +def validate_network_policy_yaml( + data: Dict[str, Any], + result: ValidationResult, +) -> None: + """Validate network_policy.yaml entries.""" + names_seen: Set[str] = set() + + for policy_name, policy_cfg in data.items(): + if not isinstance(policy_cfg, dict): + result.error( + f"Network policy {policy_name}: expected a mapping, got {type(policy_cfg).__name__}" + ) + continue + + upper_name = policy_name.upper() + if upper_name in names_seen: + result.error(f"Network policy {policy_name}: duplicate policy name") + names_seen.add(upper_name) + + # Validate CIDR entries in allowed_ip_list + for ip_entry in policy_cfg.get("allowed_ip_list", []): + try: + ipaddress.ip_network(str(ip_entry), strict=False) + except ValueError: + result.error( + f"Network policy {policy_name}: '{ip_entry}' is not valid CIDR notation" + ) + + # Also check blocked_ip_list if present + for ip_entry in policy_cfg.get("blocked_ip_list", []): + try: + ipaddress.ip_network(str(ip_entry), strict=False) + except ValueError: + result.error( + f"Network policy {policy_name}: blocked IP '{ip_entry}' is not valid CIDR notation" + ) + + count = len(data) + result.ok(f"{count} network polic{'ies' if count != 1 else 'y'} validated") + + +def validate_resource_monitor_yaml( + data: Dict[str, Any], + result: ValidationResult, +) -> None: + """Validate resource_monitor.yaml entries (basic structure check).""" + for monitor_name, monitor_cfg in data.items(): + if not isinstance(monitor_cfg, dict): + result.error( + f"Resource monitor {monitor_name}: expected a mapping, got {type(monitor_cfg).__name__}" + ) + continue + + credit_quota = monitor_cfg.get("credit_quota") + if credit_quota is not None: + try: + val = int(credit_quota) + if val <= 0: + result.error( + f"Resource monitor {monitor_name}: credit_quota must be positive, got {val}" + ) + except (ValueError, TypeError): + result.error( + f"Resource monitor {monitor_name}: credit_quota must be an integer" + ) + + count = len(data) + result.ok(f"{count} resource monitor{'s' if count != 1 else ''} validated") + + +# --------------------------------------------------------------------------- +# Output helpers +# --------------------------------------------------------------------------- + +# ANSI colour helpers for terminal output +_GREEN = "\033[32m" +_YELLOW = "\033[33m" +_RED = "\033[31m" +_RESET = "\033[0m" +_BOLD = "\033[1m" + + +def _print_file_results( + filename: str, + file_result: ValidationResult, + quiet: bool, +) -> None: + """Print validation results for a single file.""" + has_output = False + + if not quiet: + print(f"\n{_BOLD}{filename}{_RESET}") + has_output = True + + # Print info (success messages) + if not quiet: + for msg in file_result.info: + print(f" {_GREEN}\u2713{_RESET} {msg}") + + # Print warnings + if not quiet: + for msg in file_result.warnings: + print(f" {_YELLOW}\u26a0 WARNING:{_RESET} {msg}") + + # Print errors (always shown) + for msg in file_result.errors: + if not has_output: + print(f"\n{_BOLD}{filename}{_RESET}") + has_output = True + print(f" {_RED}\u2717 ERROR:{_RESET} {msg}") + + +# --------------------------------------------------------------------------- +# Main validation orchestrator +# --------------------------------------------------------------------------- + + +def run_validation(config_dir: Path, target_files: Optional[List[str]], strict: bool, quiet: bool) -> int: + """ + Run all validation checks and return exit code. + + Returns: + 0 on success (or warnings-only without --strict) + 1 on errors (or warnings with --strict) + """ + if not config_dir.is_dir(): + print(f"{_RED}Error:{_RESET} Config directory not found: {config_dir}") + return 1 + + if not quiet: + print(f"Validating {config_dir}/ configuration...") + + # ----------------------------------------------------------------------- + # 1. Load all YAML files (needed for cross-references even if targeting specific files) + # ----------------------------------------------------------------------- + + yaml_files = { + "user.yaml": config_dir / "user.yaml", + "business_role.yaml": config_dir / "business_role.yaml", + "tech_role.yaml": config_dir / "tech_role.yaml", + "warehouse.yaml": config_dir / "warehouse.yaml", + "network_policy.yaml": config_dir / "network_policy.yaml", + "resource_monitor.yaml": config_dir / "resource_monitor.yaml", + } + + loaded: Dict[str, Dict[str, Any]] = {} + syntax_result = ValidationResult() + + for name, path in yaml_files.items(): + if not path.exists(): + # Not every project will have all files; skip missing ones silently + loaded[name] = {} + continue + data, err = _load_yaml(path) + if err: + syntax_result.error(f"{name}: {err}") + loaded[name] = {} + else: + loaded[name] = data # type: ignore[assignment] + + # Print syntax errors first + if syntax_result.has_errors: + _print_file_results("YAML Syntax", syntax_result, quiet) + + # ----------------------------------------------------------------------- + # 2. Build cross-reference sets + # ----------------------------------------------------------------------- + + business_roles: Set[str] = {k.upper() for k in loaded.get("business_role.yaml", {})} + tech_roles: Set[str] = {k.upper() for k in loaded.get("tech_role.yaml", {})} + warehouses: Set[str] = {k.upper() for k in loaded.get("warehouse.yaml", {})} + resource_monitors: Set[str] = {k.upper() for k in loaded.get("resource_monitor.yaml", {})} + + # ----------------------------------------------------------------------- + # 3. Determine which files to validate + # ----------------------------------------------------------------------- + + if target_files: + # Resolve target files to their canonical names + files_to_validate: Set[str] = set() + for tf in target_files: + tf_path = Path(tf) + basename = tf_path.name + if basename in yaml_files: + files_to_validate.add(basename) + else: + # Try to match by stem + for known_name in yaml_files: + if known_name.startswith(tf_path.stem): + files_to_validate.add(known_name) + break + else: + print(f"{_YELLOW}\u26a0 WARNING:{_RESET} Unknown config file: {tf} (skipping)") + else: + files_to_validate = set(yaml_files.keys()) + + # ----------------------------------------------------------------------- + # 4. Run per-file validators + # ----------------------------------------------------------------------- + + all_results: List[Tuple[str, ValidationResult]] = [] + + validators = { + "user.yaml": lambda r: validate_user_yaml(loaded["user.yaml"], r, business_roles), + "business_role.yaml": lambda r: validate_business_role_yaml( + loaded["business_role.yaml"], r, tech_roles, warehouses + ), + "tech_role.yaml": lambda r: validate_tech_role_yaml( + loaded["tech_role.yaml"], r, warehouses + ), + "warehouse.yaml": lambda r: validate_warehouse_yaml( + loaded["warehouse.yaml"], r, resource_monitors + ), + "network_policy.yaml": lambda r: validate_network_policy_yaml( + loaded["network_policy.yaml"], r + ), + "resource_monitor.yaml": lambda r: validate_resource_monitor_yaml( + loaded["resource_monitor.yaml"], r + ), + } + + for filename in sorted(files_to_validate): + if filename not in validators: + continue + if not loaded.get(filename): + continue + + file_result = ValidationResult() + validators[filename](file_result) + all_results.append((filename, file_result)) + _print_file_results(filename, file_result, quiet) + + # ----------------------------------------------------------------------- + # 5. Cross-reference validation (only when validating all files) + # ----------------------------------------------------------------------- + + if not target_files: + xref_result = ValidationResult() + _cross_reference_checks(loaded, xref_result, business_roles, tech_roles, warehouses, resource_monitors) + if xref_result.errors or xref_result.warnings or xref_result.info: + if not quiet or xref_result.errors: + print(f"\n{_BOLD}Cross-reference checks:{_RESET}") + for msg in xref_result.info: + if not quiet: + print(f" {_GREEN}\u2713{_RESET} {msg}") + for msg in xref_result.warnings: + if not quiet: + print(f" {_YELLOW}\u26a0 WARNING:{_RESET} {msg}") + for msg in xref_result.errors: + print(f" {_RED}\u2717 ERROR:{_RESET} {msg}") + all_results.append(("cross-references", xref_result)) + + # ----------------------------------------------------------------------- + # 6. Summary + # ----------------------------------------------------------------------- + + total_errors = syntax_result.errors[:] + total_warnings = syntax_result.warnings[:] + for _, fr in all_results: + total_errors.extend(fr.errors) + total_warnings.extend(fr.warnings) + + error_count = len(total_errors) + warning_count = len(total_warnings) + + print(f"\n{_BOLD}Summary:{_RESET} ", end="") + parts = [] + if error_count: + parts.append(f"{_RED}{error_count} error{'s' if error_count != 1 else ''}{_RESET}") + if warning_count: + parts.append(f"{_YELLOW}{warning_count} warning{'s' if warning_count != 1 else ''}{_RESET}") + if not parts: + parts.append(f"{_GREEN}All checks passed{_RESET}") + print(", ".join(parts)) + + # Exit code + if error_count > 0: + return 1 + if warning_count > 0 and strict: + return 1 + return 0 + + +def _cross_reference_checks( + loaded: Dict[str, Dict[str, Any]], + result: ValidationResult, + business_roles: Set[str], + tech_roles: Set[str], + warehouses: Set[str], + resource_monitors: Set[str], +) -> None: + """Run cross-file reference integrity checks.""" + + # 1. Users' default_role -> valid roles + users = loaded.get("user.yaml", {}) + bad_default_roles = [] + for user_name, user_cfg in users.items(): + if not isinstance(user_cfg, dict): + continue + default_role = user_cfg.get("default_role") + if not default_role: + continue + role_upper = str(default_role).upper() + is_system = role_upper in SYSTEM_ROLES + is_business_raw = role_upper in business_roles + is_business_suffixed = role_upper.replace(BUSINESS_ROLE_SUFFIX, "") in business_roles + if not (is_system or is_business_raw or is_business_suffixed): + bad_default_roles.append((user_name, default_role)) + + if bad_default_roles: + for user_name, role in bad_default_roles: + result.error(f"User {user_name}: default_role '{role}' not defined as a business or system role") + else: + result.ok("All user default_roles reference valid roles") + + # 2. Business roles' tech_roles -> valid tech roles + br_data = loaded.get("business_role.yaml", {}) + bad_tech_refs = [] + for role_name, role_cfg in br_data.items(): + if not isinstance(role_cfg, dict): + continue + for tr in role_cfg.get("tech_roles", []): + if str(tr).upper() not in tech_roles: + bad_tech_refs.append((role_name, tr)) + + if bad_tech_refs: + for role_name, tr in bad_tech_refs: + result.error(f"Business role {role_name}: tech_role '{tr}' not defined in tech_role.yaml") + else: + result.ok("All business role tech_roles exist") + + # 3. Business roles' warehouse_usage -> valid warehouses + bad_wh_refs = [] + for role_name, role_cfg in br_data.items(): + if not isinstance(role_cfg, dict): + continue + for wh in role_cfg.get("warehouse_usage", []): + if str(wh).upper() not in warehouses: + bad_wh_refs.append((role_name, wh)) + + if bad_wh_refs: + for role_name, wh in bad_wh_refs: + result.error(f"Business role {role_name}: warehouse '{wh}' not defined in warehouse.yaml") + else: + result.ok("All business role warehouse_usage references exist") + + # 4. Warehouses' resource_monitor -> valid monitors + wh_data = loaded.get("warehouse.yaml", {}) + bad_monitor_refs = [] + for wh_name, wh_cfg in wh_data.items(): + if not isinstance(wh_cfg, dict): + continue + monitor = wh_cfg.get("resource_monitor") + if monitor and str(monitor).upper() not in resource_monitors: + bad_monitor_refs.append((wh_name, monitor)) + + if bad_monitor_refs: + for wh_name, monitor in bad_monitor_refs: + result.error(f"Warehouse {wh_name}: resource_monitor '{monitor}' not defined in resource_monitor.yaml") + else: + result.ok("All warehouse resource_monitors reference valid monitors") + + # 5. Tech roles' WAREHOUSE:USAGE -> valid warehouses + tr_data = loaded.get("tech_role.yaml", {}) + bad_wh_grants = [] + for role_name, role_cfg in tr_data.items(): + if not isinstance(role_cfg, dict): + continue + for section_name in ("grants", "future_grants"): + section = role_cfg.get(section_name, {}) + if not isinstance(section, dict): + continue + for grant_key, targets in section.items(): + parts = str(grant_key).split(":", 1) + if len(parts) == 2 and parts[0].upper() == "WAREHOUSE" and isinstance(targets, list): + for wh in targets: + if str(wh).upper() not in warehouses: + bad_wh_grants.append((role_name, wh)) + + if bad_wh_grants: + for role_name, wh in bad_wh_grants: + result.error(f"Tech role {role_name}: warehouse '{wh}' in grants not defined in warehouse.yaml") + else: + result.ok("All tech role warehouse grants reference valid warehouses") + + +# --------------------------------------------------------------------------- +# CLI entry point +# --------------------------------------------------------------------------- + + +def main() -> None: + """Main entry point for the validate-config command.""" + parser = argparse.ArgumentParser( + description="Validate SnowDDL YAML configuration files before deployment.", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=( + "Examples:\n" + " uv run validate-config # Validate all configs\n" + " uv run validate-config --strict # Fail on warnings too\n" + " uv run validate-config --quiet # Only show errors\n" + " uv run validate-config snowddl/user.yaml # Validate specific file\n" + ), + ) + parser.add_argument( + "files", + nargs="*", + help="Optional file path(s) to validate (default: all configs in snowddl/)", + ) + parser.add_argument( + "--strict", + action="store_true", + help="Treat warnings as errors (exit code 1 for warnings)", + ) + parser.add_argument( + "--quiet", + action="store_true", + help="Only show errors, suppress informational and warning output", + ) + parser.add_argument( + "--config-dir", + type=str, + default=None, + help="Path to the snowddl config directory (auto-detected by default)", + ) + + args = parser.parse_args() + + # Determine config directory + if args.config_dir: + config_dir = Path(args.config_dir) + else: + project_root = _find_project_root() + config_dir = project_root / "snowddl" + + target_files = args.files if args.files else None + + exit_code = run_validation(config_dir, target_files, args.strict, args.quiet) + sys.exit(exit_code) + + +if __name__ == "__main__": + main() diff --git a/src/management_cli.py b/src/management_cli.py index 143a897..4638fac 100644 --- a/src/management_cli.py +++ b/src/management_cli.py @@ -159,6 +159,16 @@ def update_user_password(): update_user_password() +# === CONFIGURATION VALIDATION === + + +def validate_config(): + """Validate SnowDDL YAML configuration files before deployment.""" + from validate_config import main + + main() + + # === STREAMLIT TESTING COMMANDS === @@ -226,6 +236,13 @@ def github_to_snowddl(): main() +def generate_terraform(): + """Generate Terraform HCL files from SnowDDL YAML configurations.""" + from generate_terraform import main + + main() + + if __name__ == "__main__": print("Use 'uv run ' where command includes:") print(" Core: warehouses, costs, security, backup, users") diff --git a/tests/integration/test_authentication_methods.py b/tests/integration/test_authentication_methods.py new file mode 100644 index 0000000..c91c3e1 --- /dev/null +++ b/tests/integration/test_authentication_methods.py @@ -0,0 +1,208 @@ +"""Integration tests for authentication methods. + +Tests RSA key operations, Fernet encryption roundtrips, and YAML password handling. +""" + +import os +import subprocess +import tempfile +import pytest +import yaml +from pathlib import Path +from cryptography.fernet import Fernet + +import sys + +sys.path.insert(0, str(Path(__file__).parent.parent.parent / "src")) + +from user_management.encryption import FernetEncryption +from user_management.yaml_handler import YAMLHandler + + +class TestRSAKeyGeneration: + """Test RSA key pair generation and validation.""" + + def test_generate_rsa_key_pair(self): + """Generate an RSA key pair using openssl and verify format.""" + with tempfile.TemporaryDirectory() as tmpdir: + private_key = Path(tmpdir) / "test_key.p8" + public_key = Path(tmpdir) / "test_key.pub" + + # Generate private key + result = subprocess.run( + [ + "bash", + "-c", + f"openssl genrsa 2048 | openssl pkcs8 -topk8 -inform PEM -nocrypt -out {private_key}", + ], + capture_output=True, + text=True, + ) + assert result.returncode == 0, f"Key generation failed: {result.stderr}" + + # Generate public key from private + result = subprocess.run( + [ + "openssl", + "rsa", + "-in", + str(private_key), + "-pubout", + "-out", + str(public_key), + ], + capture_output=True, + text=True, + ) + assert result.returncode == 0 + + # Verify files exist and have content + assert private_key.exists() + assert public_key.exists() + + pub_content = public_key.read_text() + assert "BEGIN PUBLIC KEY" in pub_content + assert "END PUBLIC KEY" in pub_content + + def test_rsa_public_key_in_yaml_format(self): + """Verify RSA public key can be stored in YAML user config format.""" + with tempfile.TemporaryDirectory() as tmpdir: + private_key = Path(tmpdir) / "test_key.p8" + public_key = Path(tmpdir) / "test_key.pub" + + subprocess.run( + [ + "bash", + "-c", + f"openssl genrsa 2048 | openssl pkcs8 -topk8 -inform PEM -nocrypt -out {private_key}", + ], + capture_output=True, + ) + subprocess.run( + ["openssl", "rsa", "-in", str(private_key), "-pubout", "-out", str(public_key)], + capture_output=True, + ) + + pub_content = public_key.read_text() + # Strip headers for SnowDDL format + lines = pub_content.strip().split("\n") + key_body = "\n".join(lines[1:-1]) + + user_config = { + "TEST_SERVICE": { + "type": "SERVICE", + "rsa_public_key": key_body, + "default_role": "TEST_ROLE", + } + } + + yaml_output = yaml.dump(user_config, default_flow_style=False) + reloaded = yaml.safe_load(yaml_output) + + assert reloaded["TEST_SERVICE"]["rsa_public_key"] == key_body + + +class TestFernetEncryptionRoundtrip: + """Test full encryption/decryption lifecycle.""" + + def test_encrypt_decrypt_roundtrip(self): + """Encrypt a password and decrypt it back.""" + key = Fernet.generate_key().decode("utf-8") + enc = FernetEncryption(key=key) + + passwords = [ + "SimplePassword123!", + "P@$$w0rd!#%^&*()", + "Unicode: รœnรฎcรถdรฉ ไฝ ๅฅฝ", + "", + "A" * 1000, + ] + + for password in passwords: + encrypted = enc.encrypt_password(password) + decrypted = enc.decrypt_password(encrypted) + assert decrypted == password, f"Roundtrip failed for: {password!r}" + + def test_key_rotation_preserves_passwords(self): + """Key rotation should re-encrypt without losing data.""" + old_key = Fernet.generate_key().decode("utf-8") + new_key = Fernet.generate_key().decode("utf-8") + + old_enc = FernetEncryption(key=old_key) + new_enc = FernetEncryption(key=new_key) + + original_passwords = { + "user1": "Password1!", + "user2": "Password2!", + "user3": "Password3!", + } + + encrypted = {user: old_enc.encrypt_password(pw) for user, pw in original_passwords.items()} + + rotated = old_enc.rotate_keys(old_key, new_key, encrypted) + + for user, original_pw in original_passwords.items(): + decrypted = new_enc.decrypt_password(rotated[user]) + assert decrypted == original_pw + + def test_wrong_key_fails(self): + """Decryption with wrong key should fail cleanly.""" + key1 = Fernet.generate_key().decode("utf-8") + key2 = Fernet.generate_key().decode("utf-8") + + enc1 = FernetEncryption(key=key1) + enc2 = FernetEncryption(key=key2) + + encrypted = enc1.encrypt_password("Secret123!") + + from user_management.encryption import InvalidEncryptedDataError + + with pytest.raises(InvalidEncryptedDataError): + enc2.decrypt_password(encrypted) + + +class TestYAMLPasswordHandling: + """Test encrypted passwords in YAML config files.""" + + def test_user_yaml_with_encrypted_password(self): + """Users with encrypted passwords should load and save correctly.""" + with tempfile.TemporaryDirectory() as tmpdir: + config_dir = Path(tmpdir) + handler = YAMLHandler(config_directory=config_dir) + + users = { + "ENCRYPTED_USER": { + "type": "PERSON", + "login_name": "ENCRYPTED_USER", + "email": "test@example.com", + "password": "!decrypt gAAAAABencryptedplaceholder", + } + } + + handler.save_users(users, backup=False) + loaded = handler.load_users() + + assert "ENCRYPTED_USER" in loaded + assert "password" in loaded["ENCRYPTED_USER"] + + def test_service_account_no_password(self): + """Service accounts should work without passwords.""" + with tempfile.TemporaryDirectory() as tmpdir: + config_dir = Path(tmpdir) + handler = YAMLHandler(config_directory=config_dir) + + users = { + "SVC_ACCOUNT": { + "type": "SERVICE", + "login_name": "SVC_ACCOUNT", + "rsa_public_key": "MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA...", + "default_role": "SERVICE_ROLE", + } + } + + handler.save_users(users, backup=False) + loaded = handler.load_users() + + assert "SVC_ACCOUNT" in loaded + assert "password" not in loaded["SVC_ACCOUNT"] + assert loaded["SVC_ACCOUNT"]["rsa_public_key"].startswith("MIIBIjAN") diff --git a/tests/integration/test_cli_commands.py b/tests/integration/test_cli_commands.py new file mode 100644 index 0000000..c1b0063 --- /dev/null +++ b/tests/integration/test_cli_commands.py @@ -0,0 +1,123 @@ +"""Integration tests for CLI commands. + +Tests that all registered UV commands are accessible and respond correctly. +""" + +import subprocess +import pytest +from pathlib import Path + +PROJECT_ROOT = Path(__file__).parent.parent.parent + + +def run_command(cmd, timeout=30): + """Run a UV command and return the result.""" + return subprocess.run( + ["uv", "run"] + cmd, + capture_output=True, + text=True, + cwd=PROJECT_ROOT, + timeout=timeout, + ) + + +class TestCoreCommands: + """Test core SnowDDL operation commands.""" + + def test_snowddl_plan_runs(self): + """snowddl-plan should execute (may fail on missing connection).""" + result = run_command(["snowddl-plan"]) + # Should not be 'command not found' (127) + assert result.returncode != 127 + + def test_snowddl_apply_has_help(self): + """snowddl-apply should be a registered command.""" + result = run_command(["snowddl-apply", "--help"]) + assert result.returncode != 127 + + def test_snowddl_validate_exists(self): + """snowddl-validate should be a registered command.""" + result = run_command(["snowddl-validate", "--help"]) + assert result.returncode != 127 + + def test_snowddl_diff_exists(self): + """snowddl-diff should be a registered command.""" + result = run_command(["snowddl-diff", "--help"]) + assert result.returncode != 127 + + +class TestUserManagementCommands: + """Test user management commands.""" + + def test_manage_users_help(self): + """manage-users --help should show usage.""" + result = run_command(["manage-users", "--help"]) + assert result.returncode == 0 + assert "usage" in result.stdout.lower() or "manage" in result.stdout.lower() + + def test_util_generate_key(self): + """util-generate-key should generate a Fernet key.""" + result = run_command(["util-generate-key"]) + # Should output a base64-encoded key or help text + assert result.returncode != 127 + + +class TestResourceManagementCommands: + """Test resource management commands.""" + + def test_manage_warehouses_help(self): + """manage-warehouses --help should show usage.""" + result = run_command(["manage-warehouses", "--help"]) + assert result.returncode == 0 + + def test_manage_costs_help(self): + """manage-costs --help should show usage.""" + result = run_command(["manage-costs", "--help"]) + assert result.returncode == 0 + + def test_manage_security_help(self): + """manage-security --help should show usage.""" + result = run_command(["manage-security", "--help"]) + assert result.returncode == 0 + + def test_manage_backup_help(self): + """manage-backup --help should show usage.""" + result = run_command(["manage-backup", "--help"]) + assert result.returncode == 0 + + +class TestMonitoringCommands: + """Test monitoring commands.""" + + def test_monitor_health_help(self): + """monitor-health --help should show usage.""" + result = run_command(["monitor-health", "--help"]) + assert result.returncode == 0 + + def test_monitor_audit_help(self): + """monitor-audit --help should show usage.""" + result = run_command(["monitor-audit", "--help"]) + assert result.returncode == 0 + + def test_monitor_metrics_help(self): + """monitor-metrics --help should show usage.""" + result = run_command(["monitor-metrics", "--help"]) + assert result.returncode == 0 + + +class TestHelpCommand: + """Test help and discovery.""" + + def test_snowtower_help(self): + """snowtower should show available commands.""" + result = run_command(["snowtower"]) + assert result.returncode == 0 + + +class TestDeploySafe: + """Test deploy-safe command.""" + + def test_deploy_safe_help(self): + """deploy-safe --help should show usage.""" + result = run_command(["deploy-safe", "--help"]) + assert result.returncode == 0 diff --git a/tests/integration/test_deployment_workflow.py b/tests/integration/test_deployment_workflow.py new file mode 100644 index 0000000..4eb7c76 --- /dev/null +++ b/tests/integration/test_deployment_workflow.py @@ -0,0 +1,203 @@ +"""Integration tests for deployment workflow. + +Tests the plan -> review -> apply lifecycle with mocked Snowflake connections. +""" + +import subprocess +import tempfile +import shutil +import pytest +import yaml +from pathlib import Path +from unittest.mock import patch, Mock + +import sys + +sys.path.insert(0, str(Path(__file__).parent.parent.parent / "src")) + +PROJECT_ROOT = Path(__file__).parent.parent.parent + + +class TestPlanWorkflow: + """Test the snowddl-plan command behavior.""" + + def test_plan_with_valid_config(self): + """Plan should run against the real snowddl/ directory.""" + result = subprocess.run( + ["uv", "run", "snowddl-plan"], + capture_output=True, + text=True, + cwd=PROJECT_ROOT, + timeout=30, + ) + # May fail due to missing Snowflake connection, but shouldn't crash + assert result.returncode != 127, "snowddl-plan command not found" + + def test_plan_output_format(self): + """Plan output should contain recognizable SnowDDL markers.""" + result = subprocess.run( + ["uv", "run", "snowddl-plan"], + capture_output=True, + text=True, + cwd=PROJECT_ROOT, + timeout=30, + ) + output = result.stdout + result.stderr + # Should have some output, even if it's an error message + assert len(output) > 0 + + +class TestDeploySafeWorkflow: + """Test the deploy-safe wrapper.""" + + def test_deploy_safe_exists(self): + """deploy-safe command should be available.""" + result = subprocess.run( + ["uv", "run", "deploy-safe", "--help"], + capture_output=True, + text=True, + cwd=PROJECT_ROOT, + timeout=30, + ) + assert result.returncode != 127 + + def test_deploy_safe_has_dry_run(self): + """deploy-safe should support --dry-run or similar safety flag.""" + result = subprocess.run( + ["uv", "run", "deploy-safe", "--help"], + capture_output=True, + text=True, + cwd=PROJECT_ROOT, + timeout=30, + ) + # Check help text exists + assert len(result.stdout + result.stderr) > 0 + + +class TestYAMLConfigLifecycle: + """Test adding/modifying/removing users through YAML.""" + + def setup_method(self): + """Create a temporary snowddl config directory.""" + self.tmpdir = tempfile.mkdtemp() + self.config_dir = Path(self.tmpdir) + + def teardown_method(self): + """Clean up temp directory.""" + shutil.rmtree(self.tmpdir, ignore_errors=True) + + def test_add_user_to_yaml(self): + """Adding a user to YAML should be readable back.""" + from user_management.yaml_handler import YAMLHandler + + handler = YAMLHandler(config_directory=self.config_dir) + + user_data = { + "NEW_ANALYST": { + "type": "PERSON", + "login_name": "NEW_ANALYST", + "email": "analyst@example.com", + "first_name": "New", + "last_name": "Analyst", + "default_role": "ANALYST_ROLE", + } + } + + handler.save_users(user_data, backup=False) + loaded = handler.load_users() + + assert "NEW_ANALYST" in loaded + assert loaded["NEW_ANALYST"]["email"] == "analyst@example.com" + + def test_modify_user_in_yaml(self): + """Modifying a user should persist changes.""" + from user_management.yaml_handler import YAMLHandler + + handler = YAMLHandler(config_directory=self.config_dir) + + # Create initial user + handler.save_users( + {"TEST_USER": {"type": "PERSON", "login_name": "TEST_USER", "email": "old@example.com"}}, + backup=False, + ) + + # Load, modify, save + users = handler.load_users() + users["TEST_USER"]["email"] = "new@example.com" + handler.save_users(users, backup=False) + + # Verify + reloaded = handler.load_users() + assert reloaded["TEST_USER"]["email"] == "new@example.com" + + def test_remove_user_from_yaml(self): + """Removing a user should persist.""" + from user_management.yaml_handler import YAMLHandler + + handler = YAMLHandler(config_directory=self.config_dir) + + handler.save_users( + { + "USER_A": {"type": "PERSON", "login_name": "USER_A"}, + "USER_B": {"type": "PERSON", "login_name": "USER_B"}, + }, + backup=False, + ) + + users = handler.load_users() + del users["USER_A"] + handler.save_users(users, backup=False) + + reloaded = handler.load_users() + assert "USER_A" not in reloaded + assert "USER_B" in reloaded + + def test_backup_before_modification(self): + """Backup should be created before modifications.""" + from user_management.yaml_handler import YAMLHandler + + handler = YAMLHandler(config_directory=self.config_dir) + + # Save initial state + handler.save_users({"ORIGINAL": {"type": "PERSON", "login_name": "ORIGINAL"}}, backup=False) + + # Modify with backup + handler.save_users({"MODIFIED": {"type": "PERSON", "login_name": "MODIFIED"}}, backup=True) + + # Check backup exists + backups = list(handler.backup_dir.glob("user_*.yaml")) + assert len(backups) >= 1 + + # Verify backup contains original data + with open(backups[0]) as f: + backup_data = yaml.safe_load(f) + assert "ORIGINAL" in backup_data + + +class TestRollbackWorkflow: + """Test rollback capabilities.""" + + def test_git_rollback_of_yaml(self): + """Verify git can rollback YAML changes (simulated).""" + with tempfile.TemporaryDirectory() as tmpdir: + config_dir = Path(tmpdir) + user_yaml = config_dir / "user.yaml" + + # Write v1 + v1 = {"USER_V1": {"type": "PERSON"}} + user_yaml.write_text(yaml.dump(v1)) + + # Write v2 + v2 = {"USER_V2": {"type": "SERVICE"}} + user_yaml.write_text(yaml.dump(v2)) + + # Verify v2 is current + current = yaml.safe_load(user_yaml.read_text()) + assert "USER_V2" in current + assert "USER_V1" not in current + + # Simulate rollback by restoring v1 + user_yaml.write_text(yaml.dump(v1)) + restored = yaml.safe_load(user_yaml.read_text()) + assert "USER_V1" in restored + assert "USER_V2" not in restored diff --git a/tests/integration/test_yaml_validation.py b/tests/integration/test_yaml_validation.py new file mode 100644 index 0000000..2992922 --- /dev/null +++ b/tests/integration/test_yaml_validation.py @@ -0,0 +1,274 @@ +"""Integration tests for YAML configuration validation. + +Tests that the actual snowddl/ YAML files are well-formed and internally consistent. +""" + +import pytest +import yaml +from pathlib import Path + +SNOWDDL_DIR = Path(__file__).parent.parent.parent / "snowddl" + +SYSTEM_ROLES = { + "ACCOUNTADMIN", + "SECURITYADMIN", + "USERADMIN", + "SYSADMIN", + "PUBLIC", + "ORGADMIN", +} + +VALID_WAREHOUSE_SIZES = { + "X-Small", + "Small", + "Medium", + "Large", + "X-Large", + "2X-Large", + "3X-Large", + "4X-Large", + "5X-Large", + "6X-Large", +} + +VALID_OBJECT_TYPES = { + "DATABASE", + "SCHEMA", + "TABLE", + "VIEW", + "WAREHOUSE", + "STAGE", + "FUNCTION", + "PROCEDURE", + "FILE_FORMAT", + "SEQUENCE", +} + + +def load_yaml(filename): + """Load a YAML file from snowddl/ with !decrypt tag support.""" + + def decrypt_constructor(loader, node): + return "!decrypt " + loader.construct_scalar(node) + + loader = yaml.SafeLoader + loader.add_constructor("!decrypt", decrypt_constructor) + filepath = SNOWDDL_DIR / filename + if not filepath.exists(): + return {} + with open(filepath) as f: + data = yaml.load(f, Loader=loader) + return data or {} + + +class TestYAMLSyntax: + """Test that all YAML files parse without errors.""" + + @pytest.mark.parametrize( + "filename", + [ + "user.yaml", + "warehouse.yaml", + "business_role.yaml", + "tech_role.yaml", + "network_policy.yaml", + "authentication_policy.yaml", + "password_policy.yaml", + "session_policy.yaml", + "resource_monitor.yaml", + ], + ) + def test_yaml_parses(self, filename): + """Each YAML config file should parse without errors.""" + data = load_yaml(filename) + assert isinstance(data, dict), f"{filename} should contain a dictionary" + + +class TestUserConfig: + """Validate user.yaml configuration.""" + + def test_all_users_have_type(self): + """Every user must have a type field.""" + users = load_yaml("user.yaml") + for username, config in users.items(): + assert "type" in config, f"User {username} missing 'type' field" + assert config["type"] in ( + "PERSON", + "SERVICE", + ), f"User {username} has invalid type: {config['type']}" + + def test_person_users_have_email(self): + """PERSON users should have an email.""" + users = load_yaml("user.yaml") + for username, config in users.items(): + if config.get("type") == "PERSON": + assert "email" in config, f"PERSON user {username} missing 'email'" + + def test_service_users_have_rsa_key(self): + """SERVICE users should have an RSA public key.""" + users = load_yaml("user.yaml") + for username, config in users.items(): + if config.get("type") == "SERVICE": + assert ( + "rsa_public_key" in config + ), f"SERVICE user {username} missing 'rsa_public_key'" + + def test_no_duplicate_users(self): + """No duplicate user names (YAML handles this natively, but verify).""" + users = load_yaml("user.yaml") + assert len(users) > 0, "user.yaml should have at least one user" + + +class TestWarehouseConfig: + """Validate warehouse.yaml configuration.""" + + def test_all_warehouses_have_size(self): + """Every warehouse must have a size.""" + warehouses = load_yaml("warehouse.yaml") + for wh_name, config in warehouses.items(): + assert "size" in config, f"Warehouse {wh_name} missing 'size'" + + def test_warehouse_sizes_are_valid(self): + """Warehouse sizes must be valid Snowflake sizes.""" + warehouses = load_yaml("warehouse.yaml") + for wh_name, config in warehouses.items(): + assert ( + config["size"] in VALID_WAREHOUSE_SIZES + ), f"Warehouse {wh_name} has invalid size: {config['size']}" + + def test_auto_suspend_is_positive(self): + """auto_suspend should be a positive integer.""" + warehouses = load_yaml("warehouse.yaml") + for wh_name, config in warehouses.items(): + if "auto_suspend" in config: + assert ( + isinstance(config["auto_suspend"], int) and config["auto_suspend"] > 0 + ), f"Warehouse {wh_name} has invalid auto_suspend: {config['auto_suspend']}" + + +class TestBusinessRoleConfig: + """Validate business_role.yaml configuration.""" + + def test_tech_roles_exist(self): + """All tech_roles referenced in business roles should exist in tech_role.yaml.""" + business_roles = load_yaml("business_role.yaml") + tech_roles = load_yaml("tech_role.yaml") + + for role_name, config in business_roles.items(): + for tech_role in config.get("tech_roles", []): + assert ( + tech_role in tech_roles + ), f"Business role {role_name} references missing tech role: {tech_role}" + + def test_warehouse_usage_exists(self): + """All warehouse_usage entries should reference existing warehouses.""" + business_roles = load_yaml("business_role.yaml") + warehouses = load_yaml("warehouse.yaml") + + for role_name, config in business_roles.items(): + for wh in config.get("warehouse_usage", []): + assert ( + wh in warehouses + ), f"Business role {role_name} references missing warehouse: {wh}" + + def test_schema_owner_format(self): + """schema_owner entries should be DB.SCHEMA format.""" + business_roles = load_yaml("business_role.yaml") + + for role_name, config in business_roles.items(): + for entry in config.get("schema_owner", []): + parts = entry.split(".") + assert ( + len(parts) == 2 + ), f"Business role {role_name} schema_owner '{entry}' not in DB.SCHEMA format" + + +class TestTechRoleConfig: + """Validate tech_role.yaml configuration.""" + + def test_grant_keys_are_valid(self): + """Grant keys must follow OBJECT_TYPE:PRIVILEGE format.""" + tech_roles = load_yaml("tech_role.yaml") + + for role_name, config in tech_roles.items(): + for grant_key in config.get("grants", {}): + parts = grant_key.split(":", 1) + assert ( + len(parts) == 2 + ), f"Tech role {role_name} grant key '{grant_key}' not in TYPE:PRIVILEGE format" + assert ( + parts[0] in VALID_OBJECT_TYPES + ), f"Tech role {role_name} has invalid object type: {parts[0]}" + + for grant_key in config.get("future_grants", {}): + parts = grant_key.split(":", 1) + assert ( + len(parts) == 2 + ), f"Tech role {role_name} future_grant key '{grant_key}' not in TYPE:PRIVILEGE format" + assert ( + parts[0] in VALID_OBJECT_TYPES + ), f"Tech role {role_name} future_grant has invalid object type: {parts[0]}" + + def test_warehouse_grants_reference_existing(self): + """WAREHOUSE:USAGE grants should reference existing warehouses.""" + tech_roles = load_yaml("tech_role.yaml") + warehouses = load_yaml("warehouse.yaml") + + for role_name, config in tech_roles.items(): + for grant_key, targets in config.get("grants", {}).items(): + if grant_key.startswith("WAREHOUSE:"): + for wh in targets or []: + assert ( + wh in warehouses + ), f"Tech role {role_name} references missing warehouse: {wh}" + + +class TestResourceMonitorConfig: + """Validate resource_monitor.yaml configuration.""" + + def test_monitors_have_credit_quota(self): + """Every monitor must have a credit_quota.""" + monitors = load_yaml("resource_monitor.yaml") + for name, config in monitors.items(): + assert "credit_quota" in config, f"Monitor {name} missing 'credit_quota'" + assert isinstance( + config["credit_quota"], (int, float) + ), f"Monitor {name} credit_quota must be numeric" + + def test_warehouse_monitors_exist(self): + """resource_monitor references in warehouse.yaml should exist.""" + warehouses = load_yaml("warehouse.yaml") + monitors = load_yaml("resource_monitor.yaml") + + for wh_name, config in warehouses.items(): + if "resource_monitor" in config: + assert ( + config["resource_monitor"] in monitors + ), f"Warehouse {wh_name} references missing monitor: {config['resource_monitor']}" + + +class TestNetworkPolicyConfig: + """Validate network_policy.yaml configuration.""" + + def test_policies_have_allowed_ips(self): + """Every network policy should have allowed_ip_list.""" + policies = load_yaml("network_policy.yaml") + for name, config in policies.items(): + assert ( + "allowed_ip_list" in config + ), f"Network policy {name} missing 'allowed_ip_list'" + assert isinstance( + config["allowed_ip_list"], list + ), f"Network policy {name} allowed_ip_list must be a list" + + def test_ip_addresses_are_cidr(self): + """IP addresses should be in CIDR notation.""" + import ipaddress + + policies = load_yaml("network_policy.yaml") + for name, config in policies.items(): + for ip in config.get("allowed_ip_list", []): + try: + ipaddress.ip_network(ip, strict=False) + except ValueError: + pytest.fail(f"Network policy {name} has invalid CIDR: {ip}") diff --git a/uv.lock b/uv.lock index b3dacd7..3a595c6 100644 --- a/uv.lock +++ b/uv.lock @@ -487,6 +487,19 @@ version = "0.9.5" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/f1/2a/8c3ac3d8bc94e6de8d7ae270bb5bc437b210bb9d6d9e46630c98f4abd20c/csscompressor-0.9.5.tar.gz", hash = "sha256:afa22badbcf3120a4f392e4d22f9fff485c044a1feda4a950ecc5eba9dd31a05", size = 237808, upload-time = "2017-11-26T21:13:08.238Z" } +[[package]] +name = "detect-secrets" +version = "1.5.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pyyaml" }, + { name = "requests" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/69/67/382a863fff94eae5a0cf05542179169a1c49a4c8784a9480621e2066ca7d/detect_secrets-1.5.0.tar.gz", hash = "sha256:6bb46dcc553c10df51475641bb30fd69d25645cc12339e46c824c1e0c388898a", size = 97351, upload-time = "2024-05-06T17:46:19.721Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4e/5e/4f5fe4b89fde1dc3ed0eb51bd4ce4c0bca406246673d370ea2ad0c58d747/detect_secrets-1.5.0-py3-none-any.whl", hash = "sha256:e24e7b9b5a35048c313e983f76c4bd09dad89f045ff059e354f9943bf45aa060", size = 120341, upload-time = "2024-05-06T17:46:16.628Z" }, +] + [[package]] name = "distlib" version = "0.4.0" @@ -2399,6 +2412,7 @@ web = [ [package.dev-dependencies] dev = [ + { name = "detect-secrets" }, { name = "faker" }, { name = "hypothesis" }, { name = "mkdocs" }, @@ -2458,6 +2472,7 @@ provides-extras = ["dev", "test", "docs", "web"] [package.metadata.requires-dev] dev = [ + { name = "detect-secrets", specifier = ">=1.5.0" }, { name = "faker", specifier = ">=18.0.0" }, { name = "hypothesis", specifier = ">=6.0.0" }, { name = "mkdocs", specifier = ">=1.6.0" },