Pierre-VF · Pierre-VF · Jul 19, 2026 · Jul 17, 2026 · Jul 17, 2026 · Jul 17, 2026
diff --git a/.github/workflows/data_update.yaml b/.github/workflows/data_update.yaml
@@ -47,7 +47,7 @@ jobs:
         run: uv sync --group=scripts --group=app
 
       - name: Running discovery
-        run: make discover
+        run: uv run discover
         env:
           GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           # Skipping authentication on Gitlab for now

diff --git a/.github/workflows/generate_listing.yaml b/.github/workflows/generate_listing.yaml
@@ -44,7 +44,7 @@ jobs:
         run: uv sync --group=scripts --group=app
 
       - name: Running discovery
-        run: make discover
+        run: uv run discover
         env:
           GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           # Skipping authentication on Gitlab for now

diff --git a/.github/workflows/repo-assist.lock.yml b/.github/workflows/repo-assist.lock.yml
diff --git a/.github/workflows/repo-assist.md b/.github/workflows/repo-assist.md
diff --git a/README.md b/README.md
@@ -107,21 +107,21 @@ Once you have completed the steps above, you can run the following commands (onl
 Typical use-cases:
 
 - To download the dataset:
-    > make download_data
+    > uv run download_data
 - To search in CLI mode (note that this is a very basic CLI):
-    > make search
+    > uv run search
 
 
 Advanced use-cases (to regenerate listings - avoid unless necessary, as this very resource intensive)
 
 - To generate an output dataset:
-    > make generate_listing
+    > uv run generate_listing
 - To add new resources:
-    > make add
+    > uv run add
 - To refresh the list of targets to be scraped:
-    > make discover
+    > uv run discover
 - To export the datasets to FTP (using the credentials from the environment):
-    > make publish
+    > uv run publish
 
 Note: the indexing is heavy and involves a series of web (and API) calls. A caching mechanism is therefore added in the implementation of the requests (with a simple SQLite database). This means that you might potentially end with a large file stored locally on your disk (currently under 500 Mb).
 

diff --git a/src/oss4climate_mcp/llm_assistant.py → dev/llm_assistant.py b/src/oss4climate_mcp/llm_assistant.py → dev/llm_assistant.py
diff --git a/docs/scraping-process.md b/docs/scraping-process.md
@@ -12,23 +12,22 @@ The scraping process manages the collection of metadata from Git hosting platfor
 TOML Index (source of truth)
     │
     ▼
-RepositoryScraper.sync_from_toml()
+RepositoryScraper.sync_from_toml()          ← per-platform sync with interruption tolerance
     │
-    ├──► Organisations table (org metadata)
+    ├──► GitHub organisations → commit after each org
+    ├──► GitLab groups        → commit after each group
+    ├──► Codeberg organisations  (org records only; no detailed scraping)
+    └──► Bitbucket projects   → commit after each project
     │
-    ├──► Repositories table (repo metadata)
+    ▼ mark_repos_inactive()                    ← prune repos removed from TOML
+RepositoryScraper.scrape_active_repos()      ← per-repo streaming within platform buckets
     │
-    ▼
-RepositoryScraper.scrape_active_repos()
-    │
-    ├──► Platform scrapers (GitHub, GitLab, Codeberg, Bitbucket)
-    │
-    ▼
-RepositoryScraper.export_*()
+    ├──► GitHub scraper (rate limit break-out)
+    ├──► GitLab scraper
+    ├──► Codeberg scraper  (NotImplementedError → logged as failure)
+    └──► Bitbucket scraper (NotImplementedError → logged as failure)
     │
-    ├──► Feather file (for backward compatibility)
-    ├──► Summary TOML
-    └──► Failures TOML
+    ▼ RepositoryScraper.export_*()             ← CSV / JSON / Feather + summary TOML + failures TOML
 ```
 
 ## Database Schema
@@ -72,7 +71,7 @@ Stores metadata about individual repositories.
 | `latest_update` | date | Last update date |
 | `last_commit` | date | Last commit date |
 | `language` | string | Dominant programming language |
-| `all_languages` | text | JSON array of all languages |
+| `all_languages` | text | JSON array of all languages (stored as JSON-encoded string) |
 | `open_pull_requests` | integer | Number of open PRs |
 | `master_branch` | string | Default branch name |
 | `readme` | text | Cleaned README content (search plaintext) |
@@ -83,7 +82,7 @@ Stores metadata about individual repositories.
 | `last_error` | string | Last error message (NULL if no error) |
 | `error_count` | integer | Number of consecutive errors (NULL if no error) |
 | `disappeared_on` | date | Date repo was marked inactive (NULL if active) |
-| `active` | boolean | Whether the repo is currently in scope |
+| `active` | boolean | Whether the repo is currently in scope (`True` by default) |
 
 ## Configuration
 
@@ -94,67 +93,87 @@ The repository database URL is configured via the `REPOS_DATABASE_URL` environme
 - **Default**: `sqlite:///.data/repos.sqlite`
 - **PostgreSQL example**: `postgresql+psycopg2://user:pass@host:5432/oss4climate_repos`
 
-### Refresh Interval
+### Refresh Interval and Cache Lifetime
 
 The default refresh interval is **28 days**. Repositories are re-scraped if:
 - They have never been scraped (`last_scraped_at IS NULL`), OR
 - Their last scrape was more than `refresh_days` ago
 
-Override with the `refresh_days` parameter:
+Override with the constructor parameters:
 
 ```python
-scraper = RepositoryScraper(refresh_days=7)  # Re-scrape every 7 days
+scraper = RepositoryScraper(refresh_days=7, cache_lifetime=None)  # Re-scrape every 7 days; use default TTL for API caching
 ```
 
+| Parameter | Default | Description |
+|-----------|---------|-------------|
+| `refresh_days` | `28` | Days since last scrape before re-scraping |
+| `cache_lifetime` | `None` (uses platform defaults) | Optional cache lifetime for HTTP responses from Git hosting APIs |
+
 ## Pipeline Stages
 
 ### 1. Sync from TOML
 
 The `sync_from_toml()` method reconciles the database with the TOML index file:
 
-1. **Read TOML**: Load the TOML index to get the current list of organisations and repositories.
-2. **Process organisations**: For each organisation/group in the TOML:
-   - Fetch metadata from the platform API
-   - Upsert into the `organisations` table
-   - Discover repositories via the platform API
-   - Upsert discovered repositories into the `repositories` table
-3. **Process explicit repositories**: For each explicitly listed repository in the TOML, upsert into the `repositories` table.
-4. **Prune inactive repositories**: Mark repositories as inactive (`active = False`, `disappeared_on = today`) if they are no longer in the TOML and not discovered from any active organisation.
+```python
+targets = ParsingTargets.from_toml(toml_path)
+```
+
+It processes each platform in sequence, committing after every successful organisation/group for interruption tolerance:
 
-**Error handling**: Organisation-level failures (API errors) are logged and the organisation is upserted with error information. The sync continues with other organisations.
+1. **GitHub** — `_sync_github_organisations()`: For each GitHub org URL from the TOML, fetch metadata via `GithubScraper.fetch_organisation_details()` and upsert into the organisations table. Then discover repos via `fetch_repositories_in_organisation()` and upsert them with organisation_id set. Commit after each successful org.
+2. **GitLab** — `_sync_gitlab_groups()`: For each GitLab group URL, fetch metadata via direct API call (`/api/v4/groups/{path}`) and discover repos via `fetch_repositories_in_group()`. Same commit-after-each pattern.
+3. **Codeberg** — `_sync_codeberg_organisations()`: Only upserts the org record with its ID (no detailed scraping yet). No error persistence on failure — just logs a warning.
+4. **Bitbucket** — `_sync_bitbucket_projects()`: Fetches project metadata via `/api/2.0/workspaces/{workspace}` and discovers repos via `fetch_repositories_in_group()`. Upserts errors on failure like GitHub/GitLab.
+
+After all platforms are synced, explicit repositories listed in the TOML (per-platform) are upserted with minimal data (`id`, `url`). Finally:
+5. **Prune inactive** — `mark_repos_inactive()` marks any active repo not found across all platform syncs as inactive (`active = False`, `disappeared_on = today`).
+
+**Error handling**: GitHub and GitLab failures log a warning, upsert the org with error info (last_error + error_count), then continue. Codeberg silently logs warnings without persisting errors to the DB. Bitbucket mirrors the GitHub/GitLab pattern of persisting errors on failure. Each platform's sync loop commits after every individual organisation/group so that partial progress survives interruptions.
 
 ### 2. Scrape Active Repositories
 
 The `scrape_active_repos()` method fetches fresh data for repositories past their refresh threshold:
 
-1. **Query DB**: Get all active repositories where `last_scraped_at IS NULL OR last_scraped_at < now() - timedelta(days=refresh_days)`.
-2. **Build targets**: Construct a `ParsingTargets` object from the repository list.
-3. **Scrape**: Use the existing `scrape_all_targets()` function to fetch data from all platforms (reusing caching, rate limiting, and platform dispatch logic).
-4. **Sync results**: For each scraped repository:
-   - On success: upsert data, set `last_scraped_at`, reset `last_error` and `error_count`
-   - On failure: update `last_error` and increment `error_count`
+1. **Query DB**: Get all active repos where `last_scraped_at IS NULL OR last_scraped_at < now() - timedelta(days=refresh_days)`.
+2. **Group by platform**: Separate repo IDs into per-platform lists (GitHub, GitLab entries as `(url, id)` pairs, Codeberg, Bitbucket). Log a breakdown of counts per platform.
+3. **Stream within each bucket** — processes repos one at a time with immediate DB commits:
+
+   - **GitHub**: For each `repo_id`, calls `GithubScraper.fetch_project_details(repo_id)`. On `RateLimitError` (403), skips all remaining GitHub repos and records errors for them. On other exceptions, sets the error in the DB via `set_repo_error()` and continues to next repo.
+   - **GitLab**: Same per-repo streaming pattern but without rate limit break-out — failures on one provider do not block others.
+   - **Codeberg / Bitbucket**: Calls their respective scrapers' `fetch_project_details()`. Both currently raise `NotImplementedError`, so every scrape attempt for these platforms will be recorded as a failure with the error message set in DB via `set_repo_error()`.
+
+4. For each successful fetch, `_project_details_to_dict()` converts the `ProjectDetails` object to a dict (serialising `all_languages` list → JSON string, converting `readme_type` enum → value), then upserts into the repositories table with `last_scraped_at`, `active = True`, and resets error fields via `reset_repo_error()`.
 
-**Rate limiting**: Each platform scraper tracks its own rate limit state. GitHub stops after 10 consecutive 403 errors.
+5. Returns `{repo_id: error_message}` dict for all failures.
+
+**Interruption tolerance**: Every repo result is committed immediately to its own session, so partial progress survives process termination or Ctrl+C. Rate limit break-out only affects the GitHub bucket — other platforms continue unaffected.
 
 ### 3. Export Results
 
-The export methods write results to files:
+The export methods write results from the database:
+
+- **`export_to_feather(output_path)`**: Reads all active repos and writes to a file based on extension:
+  - `.feather`: Binary feather format (index set to `id`)
+  - `.csv`: Semicolon-delimited CSV (`readme` column dropped)
+  - `.json`: Transposed JSON object
 
-- **`export_to_feather()`**: Reads active repositories from the DB and writes a feather file (for backward compatibility and Typesense indexing).
-- **`export_summary_toml()`**: Generates summary statistics (repository count, languages, licenses).
-- **`export_failures_toml()`**: Generates a TOML file with failure information for repositories that had errors.
+- **`export_summary_toml(output_path)`**: Generates summary statistics including repository count, organisation count, unique languages list, and licence list. Uses tomlkit for formatting with sorted keys.
+
+- **`export_failures_toml(output_path)`**: Queries active repos where `last_error IS NOT NULL`, writes `{repo_id: error_message}` pairs under a `"failures"` key in TOML format.
 
 ## Running the Scraper
 
-### From Python
+### From Python (direct)
 
 ```python
 from oss4climate.src.repository_scraper import RepositoryScraper
 
-scraper = RepositoryScraper(refresh_days=28)
+scraper = RepositoryScraper(refresh_days=28, cache_lifetime=None)
 scraper.run(
     toml_path="index.toml",
-    feather_output=".data/listing_data.feather",
+    feather_output=".data/listing_data.feather",  # or .csv / .json
     summary_output=".data/summary.toml",
     failures_output=".data/failures_scraping.toml",
 )
@@ -166,44 +185,47 @@ scraper.run(
 from oss4climate_scripts.scripts.repository_scraping import scrape_all
 
 scrape_all(
-    target_output_file=".data/listing_data.feather",
-    refresh_days=28,
+    target_output_file=None,  # defaults to FILE_OUTPUT_LISTING_FEATHER from app config
+    fail_on_issue=False,      # if True, raises on scraping failures
+    refresh_days=28,          # days between re-scrapes
 )
 ```
 
+The `scrape_all` function:
+1. Determines the output file path (defaults to feather format; CSV/JSON extensions are auto-converted internally).
+2. Runs the full pipeline via `RepositoryScraper.run()`.
+3. Formats all generated files using pre-commit tooling (`scripts.format_all_files()` and `scripts.format_individual_file()` for failures TOML).
+
 ## Data Flow to Typesense
 
-The Typesense indexer (`seed_typesense.py`) reads from the repository database:
+The Typesense indexer reads from the repository database:
 
-1. Query active repositories from the DB via `get_repos_for_typesense()`
-2. Convert to DataFrame format
-3. Mark high-quality repos (from OpenSustain Tech targets)
-4. Reset Typesense schema
-5. Index documents into Typesense
+1. Query active repositories via `get_repos_for_typesense(session)` — returns a list of dicts with fields: `id`, `name`, `description`, `readme`, `organisation_id`, `licence`, `language`, `url`, `last_commit`, `is_fork`.
+2. Index documents into Typesense for search functionality.
 
 ## Data Flow to App
 
-The app reads from Typesense (not directly from the database). The Typesense index is the app's data source. The database is the source of truth for scraped data, and Typesense is the search index.
+The app reads from Typesense (not directly from the database). The Typesense index is the app's data source. The database is the source of truth for scraped data, and Typesense serves as the search layer on top.
 
 ## Platform Support
 
 | Platform | Organisation Discovery | Repository Scraping | README Fetching |
 |----------|----------------------|---------------------|-----------------|
-| GitHub | Yes | Yes | Yes |
-| GitLab | Yes | Yes | Yes |
-| Codeberg | Partial | Partial | No (NotImplemented) |
-| Bitbucket | Partial | Partial | No (NotImplemented) |
+| GitHub | Yes (full metadata) | Yes (`fetch_project_details`) | N/A — handled by `ProjectDetails` model |
+| GitLab | Yes (via `/api/v4/groups/{path}` API) | Yes (`fetch_project_details`) | N/A — handled by `ProjectDetails` model |
+| Codeberg | Partial (org record only; no metadata fetch) | NotImplementedError → failure logged | No (`NotImplementedError`) |
+| Bitbucket | Yes (via `/api/2.0/workspaces/{workspace}` API) | NotImplementedError → failure logged | No (`fetch_repository_readme` raises `NotImplementedError`) |
 
-Codeberg and Bitbucket scrapers have limited functionality. Only GitHub and GitLab scrapers are fully implemented.
+Only GitHub and GitLab scrapers are fully implemented for repository data fetching. Codeberg organisation discovery creates org records without metadata; Bitbucket discovers repos but cannot scrape their details yet. All four platforms participate in the sync phase (TOML reconciliation), ensuring that even unsupported platform repos remain tracked as active/inactive based on TOML membership alone.
 
 ## Error Tracking
 
 Both organisations and repositories track errors:
 
-- `last_error`: The most recent error message (NULL if no error)
-- `error_count`: Number of consecutive errors (NULL if no error)
+- `last_error`: The most recent error message (`NULL` if no error)
+- `error_count`: Number of consecutive errors (`NULL` if no error; incremented by 1 per failure via `(repo.error_count or 0) + 1`)
 
-On successful scrape, both fields are reset to NULL. On failure, `last_error` is set and `error_count` is incremented.
+On successful scrape, both fields are reset to NULL. On failure, `last_error` is set and `error_count` is incremented. Error records for organisations during sync are persisted even on interruption (commit after each org/group). Repository errors from scraping are committed immediately per-repo within their platform bucket.
 
 ## Database Files
 

diff --git a/makefile b/makefile
@@ -5,9 +5,6 @@ CLI_NAME := "oss4climate_scripts.cli"
 install:
 	uv sync --all-groups
 
-.PHONY: add
-add:
-	uv run typer $(CLI_NAME) run add
 
 .PHONY: build
 build:
@@ -18,34 +15,9 @@ code_cleanup:
 	uv tool run pre-commit install
 	uv tool run pre-commit run --all
 
-.PHONY: discover
-discover:
-	uv run typer $(CLI_NAME) run discover
-
-.PHONY: generate_listing
-generate_listing:
-	# Note: typer processes "_" as "-"
-	uv run typer $(CLI_NAME) run generate-listing
-
-.PHONY: publish
-publish:
-	uv run typer $(CLI_NAME) run publish
-
-.PHONY: search
-search:
-	uv run typer $(CLI_NAME) run search
-
-.PHONY: download_data
-download_data:
-	uv run typer $(CLI_NAME) run download-data
-
 .PHONY: run_app
 run_app:
-	uv run gunicorn -k uvicorn.workers.UvicornWorker -b 0.0.0.0:8080 app:app
-
-.PHONY: help
-help:
-	uv run typer $(CLI_NAME) run --help
+	uv run gunicorn -k uvicorn.workers.UvicornWorker -b 0.0.0.0:8000 app:app
 
 .PHONY: test
 test:
@@ -59,9 +31,6 @@ test_with_coverage:
 	uv run python -m http.server 9001 --directory htmlcov/
 
 # CLI entries
-cli_help:
-	uv run typer $(CLI_NAME) run --help
-
 
 .PHONY: seed_typesense
 seed_typesense:

diff --git a/pyproject.toml b/pyproject.toml
@@ -1,6 +1,6 @@
 [project]
 name = "oss4climate"
-version = "0.10.0"
+version = "0.11.0"
 readme = "README.md"
 description = "Package to create listings of Open Source for Energy"
 authors = [
@@ -37,7 +37,6 @@ packages = [
     "src/oss4climate",
     "src/oss4climate_scripts",
     "src/oss4climate_app",
-    "src/oss4climate_mcp",
 ]
 
 [build-system]
@@ -48,7 +47,6 @@ build-backend = "hatchling.build"
 scripts = [
     "black>=26.3.1",
     "ruff>=0.15.15",
-    "typer>=0.25.1",
 ]
 test = [
     "pytest>=9.0.3",
@@ -71,3 +69,14 @@ mcp = [
     # Security only:
     "anthropic>=0.105.2"
 ]
+
+# scripts
+[project.scripts]
+add = "oss4climate_scripts.cli:add"
+discover = "oss4climate_scripts.cli:discover"
+# download_data = "oss4climate_scripts.cli:download_data"
+format = "oss4climate_scripts.cli:format"
+generate_listing = "oss4climate_scripts.cli:generate_listing"
+publish = "oss4climate_scripts.cli:publish"
+seed_typesense = "oss4climate_app.seed_typesense:seed"
+reset_typesense = "oss4climate_app.seed_typesense:reset_and_seed"