Merge pull request #33 from Chetic/pypi-package

feat: restructure as PyPI package with src layout

Author: Chetic

.github/workflows/manual-release.yml

@@ -250,13 +250,30 @@ jobs:
             --notes-file release-notes.md \
             packages/*.tar.gz
 
-      - name: Delete workflow artifacts
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+  publish_pypi:
+    needs: [prepare, release]
+    runs-on: ubuntu-latest
+    environment: pypi
+    permissions:
+      id-token: write
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+
+      - name: Set package version
         run: |
-          set -eo pipefail
-          echo "Deleting Actions Artifacts..."
-          gh api "repos/$GITHUB_REPOSITORY/actions/artifacts" --paginate -q '.artifacts[].id' | while read -r ARTIFACT_ID; do
-            echo "Deleting artifact $ARTIFACT_ID"
-            gh api "repos/$GITHUB_REPOSITORY/actions/artifacts/$ARTIFACT_ID" -X DELETE
-          done
+          VERSION="${{ needs.prepare.outputs.version }}"
+          sed -i "s/^version = .*/version = \"$VERSION\"/" pyproject.toml
+          sed -i "s/^__version__ = .*/__version__ = \"$VERSION\"/" src/chunksilo/__init__.py
+
+      - name: Build package
+        run: |
+          pip install build
+          python -m build
+
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1

.github/workflows/test-functional.yml

@@ -23,8 +23,7 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          python -m pip install -r requirements.txt
-          python -m pip install -r test/requirements.txt
+          python -m pip install -e ".[confluence,test]"
 
       - name: Check dependency licenses
         run: |
@@ -43,6 +42,7 @@ jobs:
           /tmp/license-check-venv/bin/pip install --quiet --upgrade pip
           /tmp/license-check-venv/bin/pip install --quiet \
             -r requirements.txt \
+            "llama-index-readers-confluence>=0.6.0,<1" \
             -c scripts/manylinux_2_34-constraints.txt
           /tmp/license-check-venv/bin/pip install --quiet pip-licenses
           /tmp/license-check-venv/bin/pip-licenses --partial-match \
@@ -57,7 +57,7 @@ jobs:
           DATA_DIR: ./test_data_dummy
           STORAGE_DIR: ./test_storage_dummy
         run: |
-          python index.py --download-models
+          python -m chunksilo.index --download-models
 
       - name: Run all functional tests
         env:

.github/workflows/test-rag-metrics.yml

@@ -26,8 +26,7 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          python -m pip install -r requirements.txt
-          python -m pip install -r test/requirements.txt
+          python -m pip install -e ".[confluence,test]"
 
       - name: Run RAG metrics test suite
         env:

README.md

@@ -11,7 +11,7 @@ ChunkSilo is like a local Google for your documents. It uses semantic search —
 
 ## Features
 
-- **Local indexing and search**: All indexing and search runs on your machine with bundled models — ChunkSilo itself makes no external network calls when `offline: true` (default). Note: search results are passed to your MCP client's LLM, which may be cloud-hosted.
+- **Local indexing and search**: All indexing and search runs on your machine with bundled models — ChunkSilo itself makes no external network calls when `offline: true`. Note: search results are passed to your MCP client's LLM, which may be cloud-hosted.
 - **Incremental indexing**: Only reindexes new or changed files, so re-runs are fast even on large document collections.
 - **Heading-aware navigation**: Extracts headings from PDFs, Word docs, and Markdown so results include the full heading path (e.g. "Chapter 3 > Setup > Prerequisites").
 - **Date filtering and recency boost**: Search within a date range or let recent documents rank higher automatically.
@@ -20,9 +20,29 @@ ChunkSilo is like a local Google for your documents. It uses semantic search —
 - **Confluence integration**: Optionally searches your Confluence instance alongside local files, with results returned in the same format.
 - **Source links**: Each result includes a clickable link back to the source file or Confluence page in supported MCP clients.
 
-## Quick Installation
+## Installation
 
-Download the latest release package from the [Releases page](https://github.com/Chetic/chunksilo/releases).
+### Option A: Install from PyPI (Recommended)
+
+Requires Python 3.11 or later. Models are downloaded automatically on first run (~250MB). The first run may appear to pause while models download — this is normal.
+
+```bash
+pip install chunksilo
+
+# Or with Confluence support:
+pip install chunksilo[confluence]
+```
+
+Then:
+1. **Create** a config file at `~/.config/chunksilo/config.yaml` (see [Configuration](#configuration))
+2. **Build** the index: `chunksilo --build-index`
+3. **Configure** your MCP client (see [MCP Client Configuration](#mcp-client-configuration))
+
+### Option B: Offline Bundle
+
+A self-contained package with pre-downloaded models, ideal for air-gapped environments or systems without Python installed.
+
+Download from the [Releases page](https://github.com/Chetic/chunksilo/releases):
 
 1. **Download** the `chunksilo-vX.Y.Z-manylinux_2_34_x86_64.tar.gz` file
 2. **Extract** and install:
@@ -34,7 +54,7 @@ cd chunksilo
 ```
 
 3. **Edit** `config.yaml` to set your document directories
-4. **Build** the index: `./venv/bin/python index.py`
+4. **Build** the index: `./venv/bin/chunksilo --build-index`
 5. **Configure** your MCP client (see [MCP Client Configuration](#mcp-client-configuration))
 
 ## Configuration
@@ -46,7 +66,7 @@ ChunkSilo uses a single configuration file: `config.yaml`
 Edit `config.yaml` to configure your settings:
 
 ```yaml
-# Indexing settings - used by index.py when building the search index
+# Indexing settings - used by chunksilo --build-index
 indexing:
   directories:
     - "./data"
@@ -57,12 +77,11 @@ indexing:
   chunk_size: 1600
   chunk_overlap: 200
 
-# Retrieval settings - used by chunksilo.py when searching
+# Retrieval settings - used when searching
 retrieval:
   embed_top_k: 20
   rerank_top_k: 5
   score_threshold: 0.1
-  offline: true
 
 # Confluence integration (optional)
 confluence:
@@ -80,7 +99,7 @@ All settings are optional and have sensible defaults.
 
 ### Configuration Reference
 
-#### Indexing Settings (used by index.py)
+#### Indexing Settings
 
 | Setting | Default | Description |
 | :--- | :--- | :--- |
@@ -98,7 +117,7 @@ All settings are optional and have sensible defaults.
 | `recursive` | `true` | Whether to recurse into subdirectories |
 | `enabled` | `true` | Whether to index this directory |
 
-#### Retrieval Settings (used by chunksilo.py)
+#### Retrieval Settings
 
 | Setting | Default | Description |
 | :--- | :--- | :--- |
@@ -111,10 +130,12 @@ All settings are optional and have sensible defaults.
 | `retrieval.recency_boost` | `0.3` | Recency boost weight (0.0-1.0) |
 | `retrieval.recency_half_life_days` | `365` | Days until recency boost halves |
 | `retrieval.bm25_similarity_top_k` | `10` | Files returned by BM25 filename search |
-| `retrieval.offline` | `true` | Prevent ML library network requests |
+| `retrieval.offline` | `false` | Prevent ML library network requests |
 
 #### Confluence Settings (optional)
 
+> **Note:** Confluence integration requires the optional dependency. Install with: `pip install chunksilo[confluence]`
+
 | Setting | Default | Description |
 | :--- | :--- | :--- |
 | `confluence.url` | `""` | Confluence base URL (empty = disabled) |
@@ -136,21 +157,95 @@ All settings are optional and have sensible defaults.
 | `storage.storage_dir` | `./storage` | Directory for vector index and state |
 | `storage.model_cache_dir` | `./models` | Directory for model cache |
 
+## CLI Usage
+
+The `chunksilo` command provides indexing, searching, and model management:
+
+```bash
+# Build or update the search index
+chunksilo --build-index
+
+# Search for documents
+chunksilo "your search query"
+
+# Search with date filtering
+chunksilo "quarterly report" --date-from 2024-01-01 --date-to 2024-03-31
+
+# Output results as JSON
+chunksilo "search query" --json
+
+# Show verbose output (model loading, search stats)
+chunksilo "search query" --verbose
+
+# Pre-download ML models (useful before going offline)
+chunksilo --download-models
+
+# Use a custom config file
+chunksilo --build-index --config /path/to/config.yaml
+```
+
+### CLI Options
+
+| Option | Description |
+| :--- | :--- |
+| `query` | Search query text (positional argument) |
+| `--build-index` | Build or update the search index, then exit |
+| `--download-models` | Download required ML models, then exit |
+| `--date-from` | Start date filter (YYYY-MM-DD format, inclusive) |
+| `--date-to` | End date filter (YYYY-MM-DD format, inclusive) |
+| `--json` | Output results as JSON instead of formatted text |
+| `-v, --verbose` | Show diagnostic messages (model loading, search stats) |
+| `--config` | Path to config.yaml (overrides auto-discovery) |
+
 ## MCP Client Configuration
 
 Configure your MCP client to run ChunkSilo. Below are examples for common clients.
 
-### Claude Desktop / Generic MCP Client
+> **Note:** For PyPI installs, use `chunksilo-mcp` directly. For offline bundles, use the full path `/path/to/chunksilo/venv/bin/chunksilo-mcp`. You can find the PyPI-installed binary location with `which chunksilo-mcp`.
+
+### Claude Code
+
+Add chunksilo as an MCP server using the CLI:
 
-Add to your MCP client's configuration file:
+**PyPI install:**
+```bash
+claude mcp add chunksilo --scope user -- chunksilo-mcp --config ~/.config/chunksilo/config.yaml
+```
+
+**Offline bundle:**
+```bash
+claude mcp add chunksilo --scope user -- /path/to/chunksilo/venv/bin/chunksilo-mcp --config /path/to/chunksilo/config.yaml
+```
+
+Verify it's connected:
 
+```bash
+claude mcp list
+```
+
+### Claude Desktop
+
+Add to `~/Library/Application Support/Claude/claude_desktop_config.json` (macOS) or `%APPDATA%\Claude\claude_desktop_config.json` (Windows):
+
+**PyPI install:**
+```json
+{
+  "mcpServers": {
+    "chunksilo": {
+      "command": "chunksilo-mcp",
+      "args": ["--config", "/path/to/config.yaml"]
+    }
+  }
+}
+```
+
+**Offline bundle:**
 ```json
 {
   "mcpServers": {
     "chunksilo": {
-      "command": "/path/to/chunksilo/venv/bin/python",
-      "args": ["chunksilo.py"],
-      "cwd": "/path/to/chunksilo"
+      "command": "/path/to/chunksilo/venv/bin/chunksilo-mcp",
+      "args": ["--config", "/path/to/chunksilo/config.yaml"]
     }
   }
 }
@@ -160,13 +255,27 @@ Add to your MCP client's configuration file:
 
 Add to `cline_mcp_settings.json` (typically in `~/.config/Code/User/globalStorage/saoudrizwan.claude-dev/settings/`):
 
+**PyPI install:**
+```json
+{
+  "mcpServers": {
+    "chunksilo": {
+      "command": "chunksilo-mcp",
+      "args": ["--config", "/path/to/config.yaml"],
+      "disabled": false,
+      "autoApprove": []
+    }
+  }
+}
+```
+
+**Offline bundle:**
 ```json
 {
   "mcpServers": {
     "chunksilo": {
-      "command": "/path/to/chunksilo/venv/bin/python",
-      "args": ["chunksilo.py"],
-      "cwd": "/path/to/chunksilo",
+      "command": "/path/to/chunksilo/venv/bin/chunksilo-mcp",
+      "args": ["--config", "/path/to/chunksilo/config.yaml"],
       "disabled": false,
       "autoApprove": []
     }
@@ -178,24 +287,36 @@ Add to `cline_mcp_settings.json` (typically in `~/.config/Code/User/globalStorag
 
 Add to `mcp_settings.json` (typically in `~/.config/Code/User/globalStorage/rooveterinaryinc.roo-cline/settings/`):
 
+**PyPI install:**
+```json
+{
+  "mcpServers": {
+    "chunksilo": {
+      "command": "chunksilo-mcp",
+      "args": ["--config", "/path/to/config.yaml"]
+    }
+  }
+}
+```
+
+**Offline bundle:**
 ```json
 {
   "mcpServers": {
     "chunksilo": {
-      "command": "/path/to/chunksilo/venv/bin/python",
-      "args": ["chunksilo.py"],
-      "cwd": "/path/to/chunksilo"
+      "command": "/path/to/chunksilo/venv/bin/chunksilo-mcp",
+      "args": ["--config", "/path/to/chunksilo/config.yaml"]
     }
   }
 }
 ```
 
 ## Troubleshooting
 
-- **Index missing**: Run `./venv/bin/python index.py` in the install directory.
+- **Index missing**: Run `chunksilo --build-index` (PyPI install) or `./venv/bin/chunksilo --build-index` (offline bundle).
 - **Retrieval errors**: Check paths in your MCP client configuration.
-- **Offline mode**: The release package includes models and sets `offline: true` by default. Set `retrieval.offline: false` in `config.yaml` if you need network access.
-- **Confluence Integration**: Set `confluence.url`, `confluence.username`, and `confluence.api_token` in `config.yaml` to enable Confluence search.
+- **Offline mode**: PyPI installs default to `offline: false` (models auto-download). The offline bundle includes pre-downloaded models and sets `offline: true`. Set `retrieval.offline: true` in `config.yaml` to prevent network calls after initial model download.
+- **Confluence Integration**: Install with `pip install chunksilo[confluence]`, then set `confluence.url`, `confluence.username`, and `confluence.api_token` in `config.yaml`.
 - **Custom CA Bundle**: Set `ssl.ca_bundle_path` in `config.yaml` for custom certificates.
 - **Network mounts**: Unavailable directories are skipped with a warning; indexing continues with available directories.
 - **Legacy .doc files**: Requires LibreOffice to be installed for automatic conversion to .docx. If LibreOffice is not found, .doc files are skipped with a warning. Full heading extraction is supported.