From 3cfa4acbf57cb963b50edfd9ff03323cb606ef90 Mon Sep 17 00:00:00 2001 From: Daniel Burbano Date: Sun, 20 Sep 2026 22:41:32 -0500 Subject: [PATCH 1/2] feat: enable reproducible public GitHub execution --- .env.example | 3 ++- config/repositories.evidence.json | 8 ++++++++ ingestion/common/config.py | 9 ++------- ingestion/common/environment.py | 2 +- ingestion/github/client.py | 25 +++++++++++-------------- requirements.txt | 2 +- tests/test_diagnostic_entrypoints.py | 12 ++++++------ tests/test_environment.py | 13 +++++++++++-- tests/test_github_client.py | 10 ++++++++++ 9 files changed, 52 insertions(+), 32 deletions(-) create mode 100644 config/repositories.evidence.json diff --git a/.env.example b/.env.example index 35b0dd3..86613a1 100644 --- a/.env.example +++ b/.env.example @@ -1,4 +1,5 @@ # GitHub +# Optional for public repositories; recommended for higher API rate limits. GITHUB_TOKEN= GITHUB_API_BASE_URL=https://api.github.com GITHUB_API_VERSION=2026-03-10 @@ -13,4 +14,4 @@ BIGQUERY_LOCATION=US # Application DEVFLOW_ENV=development -LOG_LEVEL=INFO \ No newline at end of file +LOG_LEVEL=INFO diff --git a/config/repositories.evidence.json b/config/repositories.evidence.json new file mode 100644 index 0000000..b609f6c --- /dev/null +++ b/config/repositories.evidence.json @@ -0,0 +1,8 @@ +{ + "repositories": [ + { + "owner": "DanteBurbano27", + "name": "devflow-engineering-analytics" + } + ] +} diff --git a/ingestion/common/config.py b/ingestion/common/config.py index 844bdf7..066bcbc 100644 --- a/ingestion/common/config.py +++ b/ingestion/common/config.py @@ -18,7 +18,7 @@ class ConfigurationError(RuntimeError): class Settings: """Runtime settings required by the ingestion application.""" - github_token: str + github_token: str | None github_api_base_url: str github_api_version: str github_timeout_seconds: float @@ -31,12 +31,7 @@ def from_env(cls) -> Settings: """Build application settings from environment variables.""" load_dotenv() - github_token = os.getenv("GITHUB_TOKEN", "").strip() - - if not github_token: - raise ConfigurationError( - "GITHUB_TOKEN is missing. Add it to the local .env file." - ) + github_token = os.getenv("GITHUB_TOKEN", "").strip() or None try: github_api_base_url = normalize_https_base_url( diff --git a/ingestion/common/environment.py b/ingestion/common/environment.py index 90bf391..2034d15 100644 --- a/ingestion/common/environment.py +++ b/ingestion/common/environment.py @@ -3,7 +3,7 @@ import sys from collections.abc import Sequence -MINIMUM_PYTHON_VERSION = (3, 11) +MINIMUM_PYTHON_VERSION = (3, 12) def is_supported_python( diff --git a/ingestion/github/client.py b/ingestion/github/client.py index 110a1a4..3516ce5 100644 --- a/ingestion/github/client.py +++ b/ingestion/github/client.py @@ -43,11 +43,11 @@ class GitHubClient: - """Client responsible for sending authenticated requests to GitHub.""" + """Client for authenticated or rate-limited public GitHub requests.""" def __init__( self, - token: str, + token: str | None = None, base_url: str = "https://api.github.com", api_version: str = "2026-03-10", timeout_seconds: float = 30.0, @@ -57,10 +57,7 @@ def __init__( sleeper: Sleeper = time.sleep, ) -> None: """Initialize the GitHub API client.""" - clean_token = token.strip() - - if not clean_token: - raise ValueError("A GitHub token is required.") + clean_token = token.strip() if token else None if timeout_seconds <= 0: raise ValueError("timeout_seconds must be greater than zero.") @@ -78,14 +75,14 @@ def __init__( self._session = session or requests.Session() self._sleeper = sleeper - self._session.headers.update( - { - "Accept": "application/vnd.github+json", - "Authorization": f"Bearer {clean_token}", - "X-GitHub-Api-Version": api_version, - "User-Agent": "devflow-engineering-analytics", - } - ) + headers = { + "Accept": "application/vnd.github+json", + "X-GitHub-Api-Version": api_version, + "User-Agent": "devflow-engineering-analytics", + } + if clean_token: + headers["Authorization"] = f"Bearer {clean_token}" + self._session.headers.update(headers) def get( self, diff --git a/requirements.txt b/requirements.txt index 8d8739f..784bfd5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,4 +4,4 @@ python-dotenv google-cloud-bigquery google-cloud-storage pandas -pyarrow \ No newline at end of file +pyarrow diff --git a/tests/test_diagnostic_entrypoints.py b/tests/test_diagnostic_entrypoints.py index cf65c73..d36a3c4 100644 --- a/tests/test_diagnostic_entrypoints.py +++ b/tests/test_diagnostic_entrypoints.py @@ -13,13 +13,13 @@ def run_diagnostic( - command: list[str], *, without_token: bool = False + command: list[str], *, invalid_base_url: bool = False ) -> subprocess.CompletedProcess[str]: """Run a diagnostic from the checkout without inheriting import helpers.""" environment = os.environ.copy() environment.pop("PYTHONPATH", None) - if without_token: - environment["GITHUB_TOKEN"] = "" + if invalid_base_url: + environment["GITHUB_API_BASE_URL"] = "http://api.github.com" return subprocess.run( [sys.executable, *command], @@ -61,10 +61,10 @@ def test_environment_diagnostic_supports_both_invocation_forms( def test_github_diagnostic_imports_in_both_invocation_forms_without_network( command: list[str], ) -> None: - """Both entrypoints must reach configuration before requiring a token.""" - result = run_diagnostic(command, without_token=True) + """Both entrypoints must reach configuration without making a request.""" + result = run_diagnostic(command, invalid_base_url=True) combined_output = result.stdout + result.stderr assert result.returncode == 1 - assert "[ERROR] GITHUB_TOKEN is missing." in result.stdout + assert "[ERROR] GitHub API base URL" in result.stdout assert "ModuleNotFoundError" not in combined_output diff --git a/tests/test_environment.py b/tests/test_environment.py index dced518..b49d032 100644 --- a/tests/test_environment.py +++ b/tests/test_environment.py @@ -8,7 +8,7 @@ def test_python_312_is_supported() -> None: """Python 3.12 must satisfy the minimum version.""" - assert is_supported_python((3, 11, 0)) is True + assert is_supported_python((3, 12, 0)) is True def test_python_313_is_supported() -> None: @@ -18,7 +18,7 @@ def test_python_313_is_supported() -> None: def test_python_311_is_not_supported() -> None: """Python 3.11 must not satisfy the minimum version.""" - assert is_supported_python((3, 10, 9)) is False + assert is_supported_python((3, 11, 9)) is False def test_current_python_runtime_is_supported() -> None: @@ -26,6 +26,15 @@ def test_current_python_runtime_is_supported() -> None: assert is_supported_python() is True +def test_settings_allow_anonymous_public_github_access( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A token is optional for public API requests with lower rate limits.""" + monkeypatch.setenv("GITHUB_TOKEN", "") + + assert Settings.from_env().github_token is None + + @pytest.mark.parametrize( "base_url", [ diff --git a/tests/test_github_client.py b/tests/test_github_client.py index cf29222..5a69b7e 100644 --- a/tests/test_github_client.py +++ b/tests/test_github_client.py @@ -233,6 +233,16 @@ def test_get_retries_after_timeout() -> None: sleeper.assert_called_once_with(1) +def test_anonymous_client_omits_authorization_header() -> None: + """Public unauthenticated requests must not send an empty credential.""" + session = Mock(spec=Session) + session.headers = {} + + GitHubClient(session=session) + + assert "Authorization" not in session.headers + + def test_get_retries_temporary_server_error() -> None: """A temporary server error must be retried.""" temporary_failure = build_response( From d89e8c1f45f689fab65e169e9c63d7f99d9b91a4 Mon Sep 17 00:00:00 2001 From: Daniel Burbano Date: Sun, 20 Sep 2026 22:41:38 -0500 Subject: [PATCH 2/2] docs: align DevFlow claims with verified capabilities --- README.md | 309 ++++++++--------------- docs/analytics/analytics_architecture.md | 4 +- docs/analytics/warehouse_models.md | 4 +- docs/architecture/system_architecture.md | 7 +- docs/execution_evidence.md | 100 ++++++++ docs/integration_acceptance_checklist.md | 8 +- docs/production_readiness.md | 4 +- docs/project_scope.md | 12 +- docs/release_checklist.md | 16 +- docs/runbook.md | 6 +- 10 files changed, 238 insertions(+), 232 deletions(-) create mode 100644 docs/execution_evidence.md diff --git a/README.md b/README.md index d9ad1db..ca091ee 100644 --- a/README.md +++ b/README.md @@ -3,251 +3,154 @@ [![Python 3.12+](https://img.shields.io/badge/python-3.12%2B-blue.svg)](https://www.python.org/downloads/) [![Code Style: Ruff](https://img.shields.io/badge/code%20style-ruff-000000.svg)](https://github.com/astral-sh/ruff) [![Tests: Pytest](https://img.shields.io/badge/tests-pytest-green.svg)](https://docs.pytest.org/) -[![Data Warehouse: BigQuery](https://img.shields.io/badge/warehouse-Google%20BigQuery-4285F4.svg)](https://cloud.google.com/bigquery) -**DevFlow Intelligence** is a V1 batch Data Engineering and Analytics platform that extracts, validates, transforms, and analyzes public GitHub repository metadata. It runs locally end to end and includes a mock-tested BigQuery adapter plus warehouse-ready SQL models. +DevFlow Intelligence is a V1 batch data pipeline for comparing maintenance activity across public GitHub repositories. It calls the GitHub REST API, preserves raw and normalized snapshots, enforces a typed contract and 16 data-quality rules, calculates repository and fleet metrics, and publishes a run manifest plus an analytics report. -It provides engineering leadership and technical leads with reliable, standardized, and auditable metrics regarding repository maintenance health, community interest, and technology stack distributions across multi-repository fleets. +## What is implemented ---- +| Capability | Status | Evidence | +|---|---|---| +| Local GitHub ingestion, normalization, quality, analytics, and partitioned JSON output | **Implemented and tested** | `orchestration/pipeline.py`, offline test suite, and [verified public API run](docs/execution_evidence.md) | +| Retry, pagination, rate-limit handling, atomic publication, and secret-safe errors | **Implemented and tested** | `ingestion/github/`, unit and regression tests | +| BigQuery insert adapter | **Implemented and mock-tested** | `ingestion/bigquery/repository_sink.py`; it is not composed into the local pipeline | +| BigQuery SQL models | **Static-tested, warehouse-ready artifacts** | `analytics/sql/`; they have not been executed against a deployed dataset | +| BigQuery infrastructure and scheduled orchestration | **Not deployed** | Optional V2 work | +| Looker Studio dashboard | **Planned V2** | No dashboard is claimed for V1 | -## Architecture Overview +## Architecture ```mermaid -flowchart TD - subgraph SOURCES["Data Sources"] - GHAPI["GitHub REST API\n(Repositories)"] - end - - subgraph INGESTION["Python Ingestion Layer"] - CLIENT["GitHub API Client\n(Auth, Pagination, Rate Limits)"] - EXTRACT["Batch Repository Extractor\n(Local Raw Persistence)"] - NORM["Metadata Normalizer\n(RepositoryMetadata)"] - end - - subgraph ANALYTICS_QUALITY["Analytics & Quality Layer"] - direction TB - CONTRACT["Data Contract Enforcement\n(RepositoryContract Schema)"] - DQ["Data Quality Engine\n(13+ Rules: ERROR / WARNING)"] - METRICS["Derived Metric Calculator\n(Recency, Status, Ratios, Scores)"] - FLEET["Portfolio Analytics Aggregator\n(Fleet Rollups & JSON Reporting)"] - end - - subgraph WAREHOUSE["BigQuery Warehouse Models"] - STG["stg_github_repositories\n(Cleaned Snapshots, FarmFingerprint)"] - INT["int_repository_activity\n(Activity Status, Safe Ratios)"] - DIM["dim_repositories\n(Conformed Dimension)"] - FCT["fct_repository_snapshots\n(Partitioned & Clustered Fact)"] - AGGS["agg_language_summary\nagg_owner_summary"] - end - - subgraph CONSUMPTION["Consumption Layer"] - DASH["Looker Studio Dashboard (Planned V2)"] - REPORTS["Automated Fleet Audit Reports"] - end - - GHAPI --> CLIENT - CLIENT --> EXTRACT - EXTRACT --> NORM - - NORM --> CONTRACT - CONTRACT --> DQ - DQ --> METRICS - METRICS --> FLEET - - FLEET --> STG - STG --> INT - INT --> DIM - INT --> FCT - DIM --> AGGS - FCT --> AGGS - - AGGS --> DASH - FLEET --> REPORTS +flowchart LR + GH[GitHub REST API] --> CLIENT[API client
auth, pagination, retries] + CLIENT --> RAW[Raw JSONL snapshot] + RAW --> NORM[Normalized repository records] + NORM --> CONTRACT[Typed contract] + CONTRACT --> DQ[16 quality rules
14 record + 2 batch] + DQ --> METRICS[Repository and fleet metrics] + METRICS --> REPORT[Analytics report + manifest] + + REPORT -. optional adapter; not wired in V1 .-> BQADAPTER[Mock-tested BigQuery adapter] + BQADAPTER -. deployment required .-> SQL[Static-tested BigQuery SQL models] + SQL -. planned V2 .-> DASH[Looker Studio] ``` ---- - -## Key Highlights & Portfolio Value - -1. **Strict Data Contract Architecture**: Enforces explicit schema types, nullability boundaries, and timezone awareness (`UTC`) via `RepositoryContract`, insulating downstream models from upstream API drift. -2. **Programmatic Data Quality Gate**: Non-destructive `DataQualityEngine` evaluating 16 record- and batch-level rules with clear severity tiers (`ERROR` vs `WARNING`), preventing corrupt records from polluting analytical marts. -3. **Deterministic Derived Metrics**: Recency calculations (`days_since_last_push`, `recency_bucket`), age (`repository_age_days`), and activity status mappings (`ACTIVE`, `STALE`, `INACTIVE`, `ARCHIVED`, `DISABLED`) execute deterministically relative to ingestion watermarks. -4. **Cloud Warehouse Readiness (Google BigQuery)**: A typed, mock-tested insert adapter and SQL models preserve snapshots, derive 64-bit `FARM_FINGERPRINT` keys, and define partitioning and clustering strategies. Real cloud execution remains environment-dependent. -5. **Dual-Team Autonomous Collaboration**: Developed under a strict isolation model where the Analytics & Quality foundation is decoupled from the Data Platform ingestion layer. -6. **Zero-Dependency Analytics Core**: Analytics and data quality engines rely on Python standard library dataclasses, enums, and typing; the complete offline suite also covers ingestion, orchestration, CI contracts, and the cloud adapter. - ---- +The solid path is publicly reproducible. The dashed path describes optional warehouse integration artifacts and future deployment work. -## Business Problem & Metric Definitions +## Reproduce the verified path -Engineering managers overseeing multiple repositories require centralized visibility into maintenance activity, open issue volumes, and repository freshness. DevFlow Intelligence provides mathematically sound and defensible answers to core questions: +Prerequisites: Python 3.12 or newer and Git. -### 1. Activity Status & Recency Classification -* **`ACTIVE`**: Repository pushed within the last 90 days, not archived, not disabled. (Note: 90/180-day thresholds are internal analytical rules, not GitHub standards). -* **`STALE`**: Repository pushed between 91 and 180 days ago. -* **`INACTIVE`**: Repository without pushes in > 180 days or without any recorded pushes. -* **`ARCHIVED`**: Explicitly marked archived by owner. -* **`DISABLED`**: Explicitly disabled repository. -* **`recency_bucket`**: Categorized time window of last push (`LAST_7_DAYS`, `LAST_30_DAYS`, `LAST_90_DAYS`, `LAST_180_DAYS`, `OVER_180_DAYS`, `NEVER_PUSHED`). - -### 2. Community & Technical Ratios -* **`fork_to_star_ratio`**: Compares the volume of repository forks to stars (does not guarantee code contribution). -* **`issue_to_star_ratio`**: Identifies repositories with disproportionate issue backlogs relative to star base. -* **`issue_to_fork_ratio`**: Relates issue volume to active fork maintenance. -* **`issue_density_per_mb`**: Evaluates backlog density normalized by repository codebase size (MB). -* **`repository_age_days`**: Elapsed lifetime of repository from creation to extraction. -* **`community_interest_score`**: Weighted composite metric: - $$\text{Score} = (\text{stars} \times 1.0) + (\text{forks} \times 2.0) + (\text{subscribers} \times 1.5)$$ -* **`size_category`**: Standardized size tiering (`EMPTY`, `MICRO`, `SMALL`, `MEDIUM`, `LARGE`). - ---- +```powershell +python -m venv .venv +.\.venv\Scripts\Activate.ps1 +python -m pip install --disable-pip-version-check -r requirements-lock.txt -## Data Quality Framework +# One public repository; no token is required. +python -m scripts.run_devflow ` + --config config/repositories.evidence.json ` + --output-root data +``` -The pipeline implements an automated data quality gate distinguishing between fatal contract breaches and operational observations: +Anonymous GitHub API access has a lower rate limit. Set `GITHUB_TOKEN` in the environment for authenticated requests; never commit it. The command prints one machine-readable result to stdout and structured operational logs to stderr. See [execution evidence](docs/execution_evidence.md) for sanitized output from an actual run. -| Severity | Action | Examples | -|---|---|---| -| **`ERROR`** | Rejects or flags record as invalid; halts analytical promotion | Negative stars/forks, invalid repository ID, empty owner/name, identity mismatch between full_name and owner, missing `extracted_at`, batch duplicates | -| **`WARNING`** | Records operational observation; allows processing | `pushed_at` earlier than `created_at`, non-standard URL scheme | -| **`INFO`** | Diagnostic tracking | Default values applied, volume distributions | +Run the same quality gates used by CI: -Run the quality diagnostic suite: ```powershell -python scripts/check_data_quality.py +python -m compileall analytics ingestion orchestration scripts +python -m ruff check . +python -m ruff format --check . +python -m pytest ``` ---- +## Key engineering decisions -## BigQuery SQL Warehouse Models +1. **Raw and normalized snapshots are separate.** Source responses remain available for audit while downstream code consumes a stable 21-field contract. +2. **Run context is shared.** One UTC extraction timestamp and run ID flow through extraction, quality, metrics, reports, and manifests. +3. **Publication is atomic and collision-safe.** Run-specific paths and a manifest-directory reservation prevent silent overwrite. +4. **Quality is deterministic.** The engine evaluates exactly 14 record rules and 2 batch rules without mutating input records. +5. **Warehouse claims stop at the tested boundary.** The BigQuery adapter uses a typed client protocol and mock tests; SQL tests validate contract shape and critical statements statically. No cloud deployment is claimed. -Located in `analytics/sql/`: +## Business problem and metric definitions -* **`staging/stg_github_repositories.sql`**: Type-casts raw snapshots, applies null handling, and generates `repository_surrogate_key` via `FARM_FINGERPRINT`. -* **`intermediate/int_repository_activity.sql`**: Calculates temporal delays and ratios with zero-safe `SAFE_DIVIDE` semantics aligned with Python. -* **`marts/dim_repositories.sql`**: Selects the latest snapshot for each conformed repository dimension row. -* **`marts/fct_repository_snapshots.sql`**: Daily snapshot fact table partitioned by `DATE(snapshot_timestamp)` and clustered by `(repository_id, language)`. -* **`marts/agg_language_summary.sql`**: Aggregated fleet metrics segmented by programming language. -* **`marts/agg_owner_summary.sql`**: Aggregated portfolio metrics segmented by repository owner. +Engineering managers comparing multiple repositories need a consistent view of maintenance activity, open-issue volume, community interest, and technology distribution. DevFlow standardizes the GitHub fields and calculates these analytical signals: ---- +- `ACTIVE`: pushed within the last 90 days and neither archived nor disabled. +- `STALE`: pushed 91 to 180 days ago. +- `INACTIVE`: no push for more than 180 days, or no recorded push. +- `ARCHIVED` and `DISABLED`: explicit GitHub repository states. +- `fork_to_star_ratio`, `issue_to_star_ratio`, and `issue_to_fork_ratio`: descriptive ratios with zero-safe behavior; they do not prove contribution or project quality. +- `issue_density_per_mb`: open-issue volume normalized by repository size. +- `community_interest_score`: a transparent portfolio heuristic, `(stars * 1.0) + (forks * 2.0) + (subscribers * 1.5)`, not a causal business metric. +- `recency_bucket`, `repository_age_days`, and `size_category`: deterministic segmentation fields. -## Repository Structure +The 90/180-day thresholds and weighted score are project-defined analytical rules, not GitHub standards. -```text -devflow-engineering-analytics/ -├── analytics/ # Analytics & Quality Layer -│ ├── contracts/ # Schema contract definitions & validation -│ │ ├── repository.py # RepositoryContract & RepositoryRecord -│ ├── metrics/ # Derived metric calculations & fleet aggregation -│ │ ├── calculator.py # RepositoryMetricCalculator -│ │ ├── definitions.py # RepositoryMetrics & ActivityStatus -│ │ └── summary.py # PortfolioAnalyticsAggregator & PortfolioSummary -│ ├── quality/ # Programmatic Data Quality Framework -│ │ ├── engine.py # DataQualityEngine -│ │ ├── models.py # QualityResult, QualityIssue, QualitySeverity -│ │ └── rules.py # 16 validation rules (Record & Batch) -│ ├── sql/ # BigQuery-ready SQL warehouse models -│ │ ├── staging/ # stg_github_repositories.sql -│ │ ├── intermediate/ # int_repository_activity.sql -│ │ ├── marts/ # dim_repositories, fct_snapshots, aggs -│ │ └── README.md # SQL modeling & optimization guide -│ └── service.py # High-level analytical facade API -├── docs/ # Technical architecture & project documentation -│ ├── analytics/ # Deep-dive analytics, DQ, and warehouse guides -│ ├── architecture/ # System architecture & ADRs -│ ├── integration_acceptance_checklist.md # Integration contract & boundaries -│ └── project_scope.md # Business problem & project scope -├── ingestion/ # GitHub API extraction client & metadata -│ ├── bigquery/ # Mock-tested BigQuery repository sink -│ ├── common/ # Config, logging, sink protocol -│ └── github/ # Client, service, batch, metadata -├── orchestration/ # Local end-to-end pipeline and stage results -├── scripts/ # Operational CLI diagnostic scripts -│ ├── check_analytics.py # Analytics pipeline diagnostic runner -│ ├── check_data_quality.py # Data quality validation diagnostic runner -│ ├── check_environment.py # Python runtime & dependencies verification -│ ├── check_github_connection.py # GitHub API connectivity test -│ └── run_devflow.py # End-to-end pipeline entrypoint -├── tests/ # Offline automated test suite -│ ├── test_analytics_*.py # Analytics service & summary unit tests -│ ├── test_contract_*.py # Data contract validation tests -│ ├── test_data_quality_*.py # Data quality rule & engine tests -│ └── test_metrics_*.py # Metric calculation & ratio tests -├── .github/workflows/ci.yml # Compile, Ruff, and pytest gates -├── pyproject.toml # Pytest & Ruff configurations -└── README.md # Main project documentation -``` +## Data quality framework ---- +The default ruleset contains exactly 16 rules: -## Getting Started +- 14 record-level checks cover identity, names, owner consistency, required-field nullability, visibility, counters, URLs, branches, and timestamp validity. +- 2 batch-level checks reject duplicate repository IDs and case-insensitive duplicate full names. +- `ERROR` issues block analytical promotion; `WARNING` issues are retained as observations. -### 1. Prerequisites -* Python $\ge 3.12$ (Tested on Python 3.14.6) -* Git +Run the diagnostic with: -### 2. Environment Setup ```powershell -# Clone or navigate to the repository -cd C:\DataEngineering\devflow-engineering-analytics +python scripts/check_data_quality.py +``` -# Create and activate virtual environment -python -m venv .venv -.\.venv\Scripts\Activate.ps1 +## Output layout + +Each run writes immutable, date- and run-partitioned files: -# Install development dependencies -python -m pip install -r requirements-dev.txt +```text +data/ +├── raw/github/repositories/extraction_date=YYYY-MM-DD/run_id=/repositories.jsonl +├── normalized/github/repositories/extraction_date=YYYY-MM-DD/run_id=/repositories.jsonl +├── analytics/github/repositories/extraction_date=YYYY-MM-DD/run_id=/report.json +└── manifests/github/repositories/extraction_date=YYYY-MM-DD/run_id=/manifest.json ``` -### 3. Verify Environment & Diagnostics -```powershell -# Run environment verification -python scripts/check_environment.py +Generated data is intentionally ignored by Git. The committed evidence document contains only a sanitized transcript, file tree, and representative aggregate report fields. -# Run analytics diagnostic -python scripts/check_analytics.py +## BigQuery boundary -# Run data quality test runner -python scripts/check_data_quality.py +`ingestion/bigquery/repository_sink.py` prepares normalized rows, converts timestamps to UTC RFC 3339 values, uses deterministic insert IDs, and sanitizes provider failures. Tests inject a mock client; the V1 orchestration does not instantiate a Google Cloud client or call this sink. -# Run extraction, quality, analytics, and manifest generation -python -m scripts.run_devflow --config config/repositories.example.json --output-root data -``` +`analytics/sql/` contains BigQuery-dialect staging, intermediate, dimension, fact, and aggregate models. Static acceptance tests check schema alignment, snapshot preservation, latest-row selection, and zero-safe ratio expressions. They do not compile or execute SQL in BigQuery. Dataset creation, permissions, deployment, scheduling, and costs remain external work. -### 4. Run Automated Test Suite -All tests execute without network calls, GitHub tokens, or cloud credentials: -```powershell -# Run the complete offline pytest suite -python -m pytest +## Repository structure -# Run Ruff linter and code formatter check -python -m ruff check . -python -m ruff format --check . +```text +analytics/ contracts, 16-rule quality engine, metrics, SQL models +ingestion/ GitHub client/service/batch persistence and BigQuery adapter +orchestration/ local end-to-end pipeline and stage results +scripts/ pipeline entrypoint, diagnostics, and benchmark +config/ safe repository-list examples +tests/ offline unit, integration, contract, regression, and acceptance tests +docs/ architecture, lineage, runbook, limitations, and execution evidence ``` ---- +## Security and reliability -## Security & Reliability Principles +- No known credentials, local `.env` file, service-account key, or generated API payload is tracked. +- Tokens are read from environment variables and redacted from structured messages and provider errors. +- Anonymous access sends no `Authorization` header. +- CI uses read-only repository permissions and disables persisted checkout credentials. +- Unit and analytical tests make no network or cloud calls; real API execution is a separate documented demonstration. -* **Zero Secret Exposure**: The repository strictly excludes credentials, `.env` files, and GCP service account keys. API tokens are ingested via environment variables (`GITHUB_TOKEN`). -* **Offline Determinism**: No network dependencies inside unit or analytical tests. Synthetic fixtures ensure 100% reproducible test runs in any CI/CD environment. -* **Idempotency**: Run IDs and deterministic output paths prevent silent local overwrite; BigQuery insert IDs provide best-effort retry deduplication. +## Development methodology ---- +The project was developed with AI-assisted parallel work across data-platform and analytics/quality concerns. That history is documented transparently in ADRs and integration notes. The portfolio evidence is the resulting code, deterministic tests, CI configuration, and reproducible execution rather than the orchestration method used to create it. -## Roadmap & Next Steps +## Roadmap -- [x] Ingestion baseline: GitHub REST API client & metadata normalization (`DFI-006.1`) -- [x] Analytics layer: Data contract enforcement & schema reflection (`RepositoryContract`) -- [x] Analytics layer: Derived recency, community interest ratios, and fleet aggregations (`PortfolioSummary`) -- [x] Data quality layer: Programmatic rule engine with ERROR/WARNING classification (16 rules) -- [x] SQL Modeling: BigQuery-ready staging, intermediate, and dimensional marts -- [x] Comprehensive offline unit, integration, contract, regression, and warehouse acceptance suite -- [x] Integration preparation: Acceptance checklist defined (`docs/integration_acceptance_checklist.md`) -- [x] Pipeline Integration: Local raw/normalized persistence, manifests, analytics reports, and mock-tested BigQuery adapter -- [x] V1 orchestration: Reproducible Python entrypoint with run context and stage status -- [x] CI Pipeline: GitHub Actions compilation, Ruff, and pytest gates -- [ ] Managed scheduling: Airflow/Dagster deployment (optional V2) -- [ ] BI Dashboard: Looker Studio analytical dashboards querying BigQuery marts (planned) +- [x] GitHub repository ingestion and metadata normalization +- [x] Typed analytical contract and 16-rule quality gate +- [x] Derived repository metrics and fleet aggregation +- [x] Raw, normalized, analytics, and manifest publication +- [x] Offline tests and GitHub Actions quality gates +- [x] Mock-tested BigQuery insert adapter and static-tested SQL models +- [ ] Execute and validate SQL against a provisioned BigQuery dataset +- [ ] Deploy managed scheduling with Airflow or Dagster +- [ ] Build the Looker Studio dashboard (planned V2) diff --git a/docs/analytics/analytics_architecture.md b/docs/analytics/analytics_architecture.md index 215100b..866f249 100644 --- a/docs/analytics/analytics_architecture.md +++ b/docs/analytics/analytics_architecture.md @@ -2,7 +2,7 @@ ## 1. Purpose and Overview -The **Analytics Layer** in DevFlow Intelligence serves as the analytical contract, quality assurance gate, and derived metrics engine bridging raw GitHub metadata ingestion and downstream cloud data warehousing (Google BigQuery) and consumption (Looker Studio - Planned V2 / APIs). +The **Analytics Layer** in DevFlow Intelligence is the implemented analytical contract, quality gate, and derived-metrics engine for GitHub metadata. BigQuery SQL and Looker Studio appear here as target-state consumers: the SQL is static-tested, while no BigQuery dataset or Looker Studio dashboard is deployed. It transforms point-in-time operational metadata into structured, reproducible analytical records with derived activity indicators, technical ratios, and fleet aggregations. @@ -53,7 +53,7 @@ flowchart TD 1. **Explicit Data Contracts**: Upstream API changes or schema drift cannot silently propagate. Records are validated against an immutable analytical contract schema (`RepositoryContract`). 2. **Non-Destructive Quality Gates**: The `DataQualityEngine` evaluates data integrity without mutating incoming records. Errors halt or isolate tainted records; warnings flag anomalies. 3. **Reproducibility & Determinism**: All temporal calculations (e.g. `days_since_last_push`) evaluate relative to `extracted_at` unless explicitly overridden, eliminating non-deterministic temporal drift across execution environments. -4. **Separation of Concerns**: Python implements contract enforcement, programmatic validation, and rapid local summary analytics. BigQuery SQL models implement warehouse-scale transformations, incremental materializations, and partitioning/clustering. +4. **Separation of Concerns**: Python implements contract enforcement, programmatic validation, and local summary analytics. The BigQuery SQL files design warehouse transformations, incremental materializations, and partitioning/clustering; live execution remains unverified. 5. **Zero External Framework Overhead**: The analytics engine utilizes standard Python library primitives (`dataclasses`, `datetime`, `enum`, `statistics`, `typing`), maximizing execution speed and eliminating brittle framework dependencies. --- diff --git a/docs/analytics/warehouse_models.md b/docs/analytics/warehouse_models.md index 2cdb340..0084bd3 100644 --- a/docs/analytics/warehouse_models.md +++ b/docs/analytics/warehouse_models.md @@ -2,7 +2,7 @@ ## 1. Overview and Architecture -DevFlow Intelligence transforms raw metadata into structured analytical layers inside **Google BigQuery**. The models are located under `analytics/sql/` and follow modern analytics engineering conventions. +This document describes the intended **Google BigQuery** warehouse shape encoded by the SQL files under `analytics/sql/`. The models are static-tested for contract shape and critical BigQuery-dialect statements; they have not been compiled or executed against a provisioned BigQuery dataset. ``` devflow_raw.raw_repositories @@ -85,4 +85,4 @@ devflow_analytics.dim_repositories devflow_analytics.fct_repository_snapshots 1. **Partition Pruning**: Always filter analytical queries on `snapshot_date` when querying `fct_repository_snapshots` to minimize BigQuery slot usage and query costs. 2. **Clustering Efficiency**: Clustering on `(repository_id, language)` ensures that multi-repository filters avoid scanning irrelevant columnar blocks. -3. **Idempotent Loads**: Deduplication via `QUALIFY ROW_NUMBER() ...` guarantees idempotency during pipeline backfills or replay runs. +3. **Designed Replay Behavior**: Deduplication via `QUALIFY ROW_NUMBER() ...` is intended to make backfills and replay runs idempotent. This remains unverified in a live BigQuery dataset. diff --git a/docs/architecture/system_architecture.md b/docs/architecture/system_architecture.md index 94b795b..e2b7cbf 100644 --- a/docs/architecture/system_architecture.md +++ b/docs/architecture/system_architecture.md @@ -1,8 +1,11 @@ # DevFlow Intelligence — System Architecture +> [!IMPORTANT] +> This document is the target-state architecture. The implemented V1 boundary is GitHub repository metadata -> local raw/normalized JSONL -> contract and 16-rule quality gate -> local analytics report and manifest. BigQuery is represented by a mock-tested adapter and static-tested SQL; GH Archive, dbt, Airflow, deployed BigQuery datasets, and Looker Studio are not implemented or deployed. + ## 1. Architecture purpose -This document describes how DevFlow Intelligence collects, stores, transforms, validates and exposes public GitHub engineering activity. +This document describes the intended architecture for collecting, storing, transforming, validating, and exposing public GitHub engineering activity. The architecture is designed as a daily batch pipeline focused on: @@ -15,7 +18,7 @@ The architecture is designed as a daily batch pipeline focused on: --- -## 2. High-level architecture +## 2. Target-state architecture ```mermaid flowchart LR diff --git a/docs/execution_evidence.md b/docs/execution_evidence.md new file mode 100644 index 0000000..7e674c3 --- /dev/null +++ b/docs/execution_evidence.md @@ -0,0 +1,100 @@ +# Verified local execution + +This evidence was captured from an actual anonymous GitHub REST API run on 2026-09-21 UTC. It exercises the implemented V1 path from API request through raw and normalized persistence, contract validation, all 16 quality rules, metric calculation, analytics publication, and manifest enrichment. It does not exercise BigQuery or Looker Studio. + +## Command + +```powershell +$env:GITHUB_TOKEN = "" +python -m scripts.run_devflow ` + --config config/repositories.evidence.json ` + --output-root data +``` + +## Safe configuration + +```json +{ + "repositories": [ + { + "owner": "DanteBurbano27", + "name": "devflow-engineering-analytics" + } + ] +} +``` + +## Sanitized console result + +The request returned HTTP 200 on its first attempt. The terminal result was: + +```json +{ + "status": "success", + "exit_code": 0, + "repositories_succeeded": 1, + "repositories_failed": 0, + "stages": [ + {"name": "configuration", "status": "success"}, + {"name": "client", "status": "success"}, + {"name": "service", "status": "success"}, + {"name": "batch", "status": "success"}, + {"name": "quality", "status": "success"}, + {"name": "analytics", "status": "success"} + ], + "error_type": null, + "error_message": null +} +``` + +The run ID and timestamps are omitted because they are generated for every execution. No token, request header, or raw API payload is included in this document. + +## Generated file structure + +```text +data/ +├── raw/github/repositories/extraction_date=2026-09-21/run_id=/repositories.jsonl +├── normalized/github/repositories/extraction_date=2026-09-21/run_id=/repositories.jsonl +├── analytics/github/repositories/extraction_date=2026-09-21/run_id=/report.json +└── manifests/github/repositories/extraction_date=2026-09-21/run_id=/manifest.json +``` + +## Representative report excerpt + +```json +{ + "meta": { + "total_input_records": 1, + "processed_records": 1 + }, + "quality": { + "summary": { + "is_valid": true, + "has_errors": false, + "has_warnings": false, + "total_records": 1, + "passed_records": 1, + "failed_records": 0, + "rules_evaluated": 16, + "error_count": 0, + "warning_count": 0 + } + }, + "portfolio": { + "summary_metrics": { + "total_repositories": 1 + }, + "activity_health": { + "active_count": 1 + }, + "languages": [ + { + "language": "Python", + "repository_count": 1 + } + ] + } +} +``` + +The full generated files remain local and ignored because GitHub API responses are mutable snapshots. This stable excerpt proves the stage outcomes and output contract without presenting volatile repository counters as permanent portfolio results. diff --git a/docs/integration_acceptance_checklist.md b/docs/integration_acceptance_checklist.md index f883b60..839ff16 100644 --- a/docs/integration_acceptance_checklist.md +++ b/docs/integration_acceptance_checklist.md @@ -126,7 +126,7 @@ The analytics layer accepts either `ingestion.github.repository_metadata.Reposit ## 6. BigQuery SQL Warehouse Models -All SQL models in `analytics/sql/` are validated for Google Cloud BigQuery compatibility: +The SQL models in `analytics/sql/` are static-tested for expected schema references and selected BigQuery-dialect statements. They have not been compiled or executed by BigQuery: 1. **`stg_github_repositories.sql`** (Staging): - Type casting and null coalescing. @@ -150,7 +150,7 @@ All SQL models in `analytics/sql/` are validated for Google Cloud BigQuery compa - [x] **Test Isolation**: 100% offline, zero network requests, zero GitHub tokens, zero cloud credentials. - [x] **Synthetic Fixtures**: All tests run with deterministic synthetic data fixtures. -- [x] **Pytest Pass**: All 109+ tests pass in $< 1.0\text{s}$. +- [x] **Pytest Pass**: All 155 collected tests pass locally on Python 3.12. - [x] **Linter Compliance**: `ruff check .` passes with zero errors or warnings. - [x] **Formatting Compliance**: `ruff format --check .` passes with line length $\le 88$. @@ -158,8 +158,8 @@ All SQL models in `analytics/sql/` are validated for Google Cloud BigQuery compa ## 8. Security & Environment Governance -- [x] **Zero Hardcoded Secrets**: No API tokens, passwords, or GCP service account keys in codebase. -- [x] **Environment Variable Ingestion**: Relies strictly on `GITHUB_TOKEN` from environment. +- [x] **Credential Scan**: No known API tokens, passwords, or GCP service-account keys are tracked. +- [x] **Environment Variable Ingestion**: `GITHUB_TOKEN` is optional for public repositories and is read from the environment when authenticated rate limits are needed. - [x] **Standard Library Core**: Analytics engine requires zero external packages beyond standard Python library, minimizing supply chain vulnerabilities. --- diff --git a/docs/production_readiness.md b/docs/production_readiness.md index 31cf0fe..05c2819 100644 --- a/docs/production_readiness.md +++ b/docs/production_readiness.md @@ -1,7 +1,7 @@ -# Production Readiness Audit +# Implementation Readiness Audit ## Audit Summary -This audit certifies the status of the DevFlow Intelligence pipeline components handled by the Antigravity Analytics team. +This audit records the test-backed implementation status of DevFlow Intelligence. It is not a production deployment or operational certification. ### 1. Data Contracts - **Status**: **READY** diff --git a/docs/project_scope.md b/docs/project_scope.md index 293383f..c653ec3 100644 --- a/docs/project_scope.md +++ b/docs/project_scope.md @@ -40,19 +40,19 @@ DevFlow Intelligence will consolidate this information into structured analytica --- -## 3. Project objective +## 3. Target-state objective -Build a reproducible batch data pipeline that: +The broader roadmap would build a reproducible batch data pipeline that: 1. Extracts public repository activity from GitHub. 2. Preserves raw source data for traceability. -3. Loads the data into Google BigQuery. -4. Cleans and transforms the data using dbt. +3. Loads the data into Google BigQuery (not deployed in V1). +4. Cleans and transforms the data using dbt (not implemented in V1). 5. Creates a basic dimensional model. 6. Generates software delivery metrics. 7. Validates data quality. -8. Automates pipeline execution using Apache Airflow. -9. Presents the results through an analytical dashboard. +8. Automates pipeline execution using Apache Airflow (planned V2). +9. Presents the results through an analytical dashboard (planned V2). --- diff --git a/docs/release_checklist.md b/docs/release_checklist.md index 7753aaf..dd3a26f 100644 --- a/docs/release_checklist.md +++ b/docs/release_checklist.md @@ -1,15 +1,15 @@ # Release Checklist -## Pre-Release (Antigravity Analytics) -- [ ] Ensure 100% test coverage using `py -3.14 -m pytest`. -- [ ] Validate linting rules using `py -3.14 -m ruff check .`. -- [ ] Confirm code formatting using `py -3.14 -m ruff format --check .`. +## Pre-Release (Analytics) +- [ ] Run the complete collected test suite using Python 3.12 or newer: `python -m pytest`. +- [ ] Validate linting rules using `python -m ruff check .`. +- [ ] Confirm code formatting using `python -m ruff format --check .`. - [ ] Run Integration Simulation (`tests/test_integration_simulation.py`) against Golden Data. -- [ ] Verify BigQuery SQL scripts compile cleanly. +- [ ] When a BigQuery project is available, compile and execute SQL against a disposable dataset; V1 tests are static only. ## Release Execution -- [ ] Merge `team/antigravity-analytics-quality` to integration branch. -- [ ] Validate downstream Codex outputs using the simulation constraints. +- [ ] Review the release diff and CI result on the release PR. +- [ ] Validate normalized outputs using the simulation constraints. ## Post-Release -- [ ] Monitor metric deviation and quality error rates for the first 24 hours. +- [ ] For a deployed scheduler, define and validate monitoring before enabling recurring runs. diff --git a/docs/runbook.md b/docs/runbook.md index 662c060..33d5b66 100644 --- a/docs/runbook.md +++ b/docs/runbook.md @@ -1,12 +1,12 @@ # Operational Runbook ## Overview -This runbook covers the standard operating procedures for the Antigravity Analytics plane in DevFlow Intelligence. +This runbook separates current V1 local operations from target-state warehouse operations. ## Daily Operations - Monitor execution logs for `ContractValidationError`. A spike in these errors indicates an upstream change in the GitHub API or Codex ingestion logic. -- Verify that `DataQualityEngine` reject rates remain stable (historically < 1%). -- Check BigQuery Dataform/dbt runs for the models in `analytics/sql/`. +- Review `DataQualityEngine` errors and warnings against the 16-rule catalog; no historical reject-rate baseline is claimed. +- **Target state only:** after a BigQuery deployment exists, monitor the jobs that execute models in `analytics/sql/`. V1 has no deployed Dataform/dbt run. ## Service Level Objectives (SLOs) - **TARGET (Phase 3)**: 99.9% of valid records processed within 5 seconds.