From 46e987e4e66eedbb69ee1185e0346b513f09f867 Mon Sep 17 00:00:00 2001 From: Torsten Mahr Date: Mon, 24 Aug 2026 23:26:23 +0200 Subject: [PATCH 1/3] feat(packaging)!: publish as trsdn-markitdown-mcp via PyPI Trusted Publishing The PyPI name `markitdown-mcp` is owned by Microsoft's official project, so this independent server is published as `trsdn-markitdown-mcp`. - rename the distribution to `trsdn-markitdown-mcp` (import package `markitdown_mcp` and the `markitdown-mcp` command are unchanged) - add a `trsdn-markitdown-mcp` console-script alias and `__main__.py` so the server can be started via `uvx` or `python -m markitdown_mcp` - complete publish metadata (SPDX license, license-files, maintainers, classifiers, keywords, project URLs) - rewrite release.yml around a `build` + `publish` pair using OIDC Trusted Publishing (environment `pypi`, `id-token: write`, no API tokens) with `twine check` and a wheel smoke test before publishing - prune tests/docs/examples/scripts/schemas from the sdist - update README, docs and install script to the new distribution name and add a prominent "not Microsoft's official package" notice BREAKING CHANGE: the PyPI distribution name changes from `markitdown-mcp` to `trsdn-markitdown-mcp`. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/workflows/pre-release.yml | 12 +- .github/workflows/release.yml | 544 +++++------------------------- CHANGELOG.md | 16 + MANIFEST.in | 19 +- README.md | 100 ++++-- RELEASE.md | 46 ++- docs/guides/KNOWN_ISSUES.md | 2 +- docs/index.rst | 4 +- markitdown_mcp/__main__.py | 6 + pyproject.toml | 30 +- scripts/install-all-deps.sh | 10 +- 11 files changed, 264 insertions(+), 525 deletions(-) create mode 100644 markitdown_mcp/__main__.py diff --git a/.github/workflows/pre-release.yml b/.github/workflows/pre-release.yml index 3dec991..41978b2 100644 --- a/.github/workflows/pre-release.yml +++ b/.github/workflows/pre-release.yml @@ -170,7 +170,7 @@ jobs: changelog.append('') changelog.append('### ๐Ÿงช Testing') - changelog.append('- Install: \`pip install markitdown-mcp==$VERSION\`') + changelog.append('- Install: \`pip install trsdn-markitdown-mcp==$VERSION\`') changelog.append('- Feedback: Report issues in GitHub with pre-release label') changelog.append('') @@ -345,7 +345,7 @@ jobs: # Install from Test PyPI pip install --index-url https://test.pypi.org/simple/ \ --extra-index-url https://pypi.org/simple/ \ - markitdown-mcp==$VERSION + trsdn-markitdown-mcp==$VERSION # Test basic functionality python -c " @@ -383,7 +383,7 @@ jobs: This is a pre-release version for testing purposes. ๐Ÿงช **Testing Instructions:** - - Install: pip install markitdown-mcp==${{ needs.prepare-prerelease.outputs.version }} + - Install: pip install trsdn-markitdown-mcp==${{ needs.prepare-prerelease.outputs.version }} - Test your workflows with this version - Report any issues with the 'pre-release' label @@ -404,7 +404,7 @@ jobs: ### Installation ```bash - pip install markitdown-mcp==$VERSION + pip install trsdn-markitdown-mcp==$VERSION ``` ### Testing Checklist @@ -453,7 +453,7 @@ jobs: VERSION="${{ needs.prepare-prerelease.outputs.version }}" # Install from PyPI - pip install markitdown-mcp==$VERSION + pip install trsdn-markitdown-mcp==$VERSION # Comprehensive functionality test python -c " @@ -498,7 +498,7 @@ jobs: echo "" echo "๐Ÿ“ฆ **Version**: $TAG" echo "๐Ÿ”— **Release**: https://github.com/${{ github.repository }}/releases/tag/$TAG" - echo "๐Ÿ“ฅ **Install**: pip install markitdown-mcp==$VERSION" + echo "๐Ÿ“ฅ **Install**: pip install trsdn-markitdown-mcp==$VERSION" echo "" if [[ "${{ needs.create-prerelease.result }}" == "success" ]]; then diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 65526ff..8e349a0 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -3,12 +3,11 @@ name: Release on: push: tags: - - 'v*.*.*' + - 'v*' + workflow_dispatch: permissions: - contents: write - id-token: write - packages: write + contents: read # Ensure only one release runs at a time concurrency: @@ -17,372 +16,99 @@ concurrency: jobs: # ========================================== - # Pre-Release Validation + # Build sdist + wheel # ========================================== - validate-release: - name: Validate Release + build: + name: Build distributions runs-on: ubuntu-latest - outputs: - version: ${{ steps.version.outputs.version }} - tag: ${{ steps.version.outputs.tag }} - is-prerelease: ${{ steps.version.outputs.prerelease }} steps: - uses: actions/checkout@v4 with: fetch-depth: 0 - - name: Extract version from tag - id: version - run: | - TAG=${GITHUB_REF#refs/tags/} - VERSION=${TAG#v} - - echo "tag=${TAG}" >> $GITHUB_OUTPUT - echo "version=${VERSION}" >> $GITHUB_OUTPUT - - # Check if this is a prerelease (contains rc, alpha, beta) - if [[ $VERSION == *"rc"* ]] || [[ $VERSION == *"alpha"* ]] || [[ $VERSION == *"beta"* ]]; then - echo "prerelease=true" >> $GITHUB_OUTPUT - else - echo "prerelease=false" >> $GITHUB_OUTPUT - fi - - - name: Validate tag format - run: | - TAG="${{ steps.version.outputs.tag }}" - if [[ ! $TAG =~ ^v[0-9]+\.[0-9]+\.[0-9]+(-.*)?$ ]]; then - echo "โŒ Invalid tag format: $TAG" - echo "Expected format: v1.2.3 or v1.2.3-rc.1" - exit 1 - fi - echo "โœ… Valid tag format: $TAG" - - - name: Check if tag already exists in releases - run: | - TAG="${{ steps.version.outputs.tag }}" - if gh release view "$TAG" >/dev/null 2>&1; then - echo "โŒ Release for tag $TAG already exists" - exit 1 - fi - echo "โœ… New release tag: $TAG" - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - # ========================================== - # Quality Gates - Must Pass - # ========================================== - quality-gates: - name: Release Quality Gates - runs-on: ubuntu-latest - needs: validate-release - strategy: - matrix: - python-version: ['3.10', '3.11', '3.12'] - steps: - - uses: actions/checkout@v4 - - - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - cache: 'pip' - - - name: Install dependencies - run: | - pip install --upgrade pip - pip install -e ".[dev]" - - - name: Format check - run: ruff format --check - - - name: Lint check - run: ruff check - - - name: Type check - run: mypy markitdown_mcp/ --strict - - - name: Run tests with coverage - run: | - pytest --cov=markitdown_mcp --cov-report=json --cov-fail-under=80 \ - tests/unit/ tests/integration/test_mcp_protocol_smoke.py - - - name: MCP protocol validation - run: | - python scripts/generate-schemas.py - - # Verify MCP server starts and responds - timeout 30 python -c " - import asyncio - from markitdown_mcp.server import MarkItDownMCPServer, MCPRequest - - async def test(): - server = MarkItDownMCPServer() - req = MCPRequest(id='test', method='tools/list', params={}) - resp = await server.handle_request(req) - assert resp.result, 'MCP server failed to respond' - tools = resp.result['tools'] - assert len(tools) >= 3, f'Expected 3+ tools, got {len(tools)}' - print(f'โœ… MCP validation passed: {len(tools)} tools available') - - asyncio.run(test()) - " - - # ========================================== - # Security Validation - # ========================================== - security-scan: - name: Security Validation - runs-on: ubuntu-latest - needs: validate-release - steps: - - uses: actions/checkout@v4 - - - uses: actions/setup-python@v5 - with: - python-version: '3.12' - - - name: Install security tools - run: | - pip install bandit safety - - - name: Security scan with Bandit - run: | - bandit -r markitdown_mcp/ -f json -o bandit-results.json - python -c " - import json - with open('bandit-results.json') as f: - results = json.load(f) - high_issues = [r for r in results.get('results', []) if r['issue_severity'] == 'HIGH'] - if high_issues: - print(f'โŒ Found {len(high_issues)} HIGH severity security issues') - for issue in high_issues[:3]: - print(f' - {issue[\"test_name\"]}: {issue[\"issue_text\"]}') - exit(1) - print('โœ… No high-severity security issues found') - " - - - name: Dependency vulnerability scan - run: | - safety check --json || true - - # ========================================== - # Build and Test Package - # ========================================== - build-package: - name: Build Package - runs-on: ubuntu-latest - needs: [validate-release, quality-gates, security-scan] - outputs: - package-name: ${{ steps.build.outputs.package-name }} - package-version: ${{ steps.build.outputs.package-version }} - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 with: python-version: '3.12' - - name: Install build tools - run: | - pip install build twine tomli_w + - name: Install build tooling + run: python -m pip install --upgrade pip build twine - - name: Build package - id: build + - name: Sync version with tag + if: startsWith(github.ref, 'refs/tags/v') run: | - # Update version in pyproject.toml to match tag - VERSION="${{ needs.validate-release.outputs.version }}" - sed -i "s/version = \".*\"/version = \"$VERSION\"/" pyproject.toml - - # Build package - python -m build + TAG="${GITHUB_REF#refs/tags/}" + VERSION="${TAG#v}" + if [[ ! "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+([.-].*)?$ ]]; then + echo "::error::Invalid tag format: $TAG (expected vX.Y.Z)" + exit 1 + fi + python - "$VERSION" <<'PY' + import pathlib + import re + import sys - # Extract package info - PACKAGE_NAME=$(python -c "import tomllib; print(tomllib.load(open('pyproject.toml', 'rb'))['project']['name'])") - echo "package-name=$PACKAGE_NAME" >> $GITHUB_OUTPUT - echo "package-version=$VERSION" >> $GITHUB_OUTPUT + version = sys.argv[1] + path = pathlib.Path("pyproject.toml") + text = path.read_text(encoding="utf-8") + text, count = re.subn( + r'^version = ".*"$', f'version = "{version}"', text, count=1, flags=re.M + ) + if count != 1: + raise SystemExit("Could not update version in pyproject.toml") + path.write_text(text, encoding="utf-8") + print(f"pyproject.toml version set to {version}") + PY - echo "๐Ÿ“ฆ Built package: $PACKAGE_NAME v$VERSION" + - name: Build sdist and wheel + run: python -m build - - name: Verify package + - name: Check distributions run: | - # Check package contents twine check dist/* - - # Install and test package - pip install dist/*.whl - - # Test basic functionality - python -c " - import markitdown_mcp - print(f'โœ… Package installed successfully: {markitdown_mcp.__version__}') + echo "### Built artifacts" >> "$GITHUB_STEP_SUMMARY" + find dist -type f -printf '%f (%s bytes)\n' | tee -a "$GITHUB_STEP_SUMMARY" + + - name: Smoke test wheel + run: | + python -m venv /tmp/smoke + /tmp/smoke/bin/pip install --upgrade pip + /tmp/smoke/bin/pip install dist/*.whl + /tmp/smoke/bin/python -c "import markitdown_mcp; print('import ok')" + REQUEST='{"jsonrpc":"2.0","id":1,"method":"tools/list","params":{}}' + echo "$REQUEST" | timeout 60 /tmp/smoke/bin/markitdown-mcp | tee /tmp/console-script.json + echo "$REQUEST" | timeout 60 /tmp/smoke/bin/trsdn-markitdown-mcp > /dev/null + echo "$REQUEST" | timeout 60 /tmp/smoke/bin/python -m markitdown_mcp > /dev/null + /tmp/smoke/bin/python -c " + import json, pathlib, sys + data = json.loads(pathlib.Path('/tmp/console-script.json').read_text()) + tools = data['result']['tools'] + assert len(tools) >= 3, f'expected 3+ tools, got {len(tools)}' + print(f'MCP entrypoint OK: {len(tools)} tools') " - # Test MCP server - echo '{}' | markitdown-mcp || echo "โš ๏ธ MCP server test skipped" - - - name: Upload build artifacts + - name: Upload distributions uses: actions/upload-artifact@v4 with: - name: dist-${{ needs.validate-release.outputs.version }} + name: dist path: dist/ + if-no-files-found: error retention-days: 30 # ========================================== - # Generate Changelog + # Publish to PyPI via Trusted Publishing (OIDC) # ========================================== - generate-changelog: - name: Generate Changelog - runs-on: ubuntu-latest - needs: validate-release - outputs: - changelog: ${{ steps.changelog.outputs.changelog }} - release-notes: ${{ steps.changelog.outputs.release-notes }} - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Generate changelog from commits - id: changelog - run: | - # Get previous tag for changelog generation - CURRENT_TAG="${{ needs.validate-release.outputs.tag }}" - PREVIOUS_TAG=$(git describe --tags --abbrev=0 $CURRENT_TAG^ 2>/dev/null || echo "") - - if [ -z "$PREVIOUS_TAG" ]; then - echo "First release - using all commits" - COMMIT_RANGE="HEAD" - else - echo "Previous tag: $PREVIOUS_TAG" - COMMIT_RANGE="$PREVIOUS_TAG..$CURRENT_TAG" - fi - - # Generate changelog from conventional commits - python -c " - import subprocess - import re - from datetime import datetime - - # Get commits in range - result = subprocess.run(['git', 'log', '--oneline', '--pretty=format:%s', '$COMMIT_RANGE'], - capture_output=True, text=True) - commits = result.stdout.strip().split('\n') if result.stdout.strip() else [] - - # Parse conventional commits - features = [] - fixes = [] - breaking = [] - other = [] - - for commit in commits: - if not commit.strip(): - continue - - # Check for breaking changes - if 'BREAKING CHANGE' in commit.upper(): - breaking.append(commit) - elif commit.startswith('feat'): - features.append(commit) - elif commit.startswith('fix'): - fixes.append(commit) - elif commit.startswith(('docs', 'test', 'chore', 'ci', 'refactor', 'perf')): - other.append(commit) - - # Generate changelog - changelog = [] - changelog.append(f'## [{\"${{ needs.validate-release.outputs.version }}\"}] - {datetime.now().strftime(\"%Y-%m-%d\")}') - changelog.append('') - - if breaking: - changelog.append('### โš ๏ธ BREAKING CHANGES') - for commit in breaking: - desc = re.sub(r'^[^:]+:', '', commit).strip() - changelog.append(f'- {desc}') - changelog.append('') - - if features: - changelog.append('### โœจ Features') - for commit in features: - desc = re.sub(r'^feat[^:]*:', '', commit).strip() - changelog.append(f'- {desc}') - changelog.append('') - - if fixes: - changelog.append('### ๐Ÿ› Bug Fixes') - for commit in fixes: - desc = re.sub(r'^fix[^:]*:', '', commit).strip() - changelog.append(f'- {desc}') - changelog.append('') - - if other: - changelog.append('### ๐Ÿ”ง Other Changes') - for commit in other[:5]: # Limit other changes - desc = re.sub(r'^[^:]+:', '', commit).strip() - changelog.append(f'- {desc}') - if len(other) > 5: - changelog.append(f'- ... and {len(other) - 5} more changes') - changelog.append('') - - changelog_text = '\n'.join(changelog) - - # Create release notes (shorter version for GitHub) - release_notes = [] - if breaking: - release_notes.append('### โš ๏ธ BREAKING CHANGES') - for commit in breaking[:3]: - desc = re.sub(r'^[^:]+:', '', commit).strip() - release_notes.append(f'- {desc}') - release_notes.append('') - - if features: - release_notes.append('### โœจ New Features') - for commit in features[:5]: - desc = re.sub(r'^feat[^:]*:', '', commit).strip() - release_notes.append(f'- {desc}') - if len(features) > 5: - release_notes.append(f'- ... and {len(features) - 5} more features') - release_notes.append('') - - if fixes: - release_notes.append('### ๐Ÿ› Bug Fixes') - for commit in fixes[:5]: - desc = re.sub(r'^fix[^:]*:', '', commit).strip() - release_notes.append(f'- {desc}') - if len(fixes) > 5: - release_notes.append(f'- ... and {len(fixes) - 5} more fixes') - release_notes.append('') - - release_notes.append('### ๐Ÿ“Š Release Info') - release_notes.append(f'- **Total commits**: {len(commits)}') - release_notes.append(f'- **Features**: {len(features)}') - release_notes.append(f'- **Bug fixes**: {len(fixes)}') - if breaking: - release_notes.append(f'- **Breaking changes**: {len(breaking)}') - - release_notes_text = '\n'.join(release_notes) - - # Output to GitHub Actions - import os - with open(os.environ['GITHUB_OUTPUT'], 'a') as f: - f.write(f'changelog< CHANGELOG.md - - # Clean up - rm CHANGELOG.md.bak - - - name: Commit changelog update - run: | - git config --local user.email "action@github.com" - git config --local user.name "GitHub Action" - - if git diff --exit-code CHANGELOG.md; then - echo "No changelog changes to commit" - else - git add CHANGELOG.md - git commit -m "docs: update changelog for v${{ needs.validate-release.outputs.version }}" - git push origin main - echo "โœ… Changelog updated" + TAG="${GITHUB_REF#refs/tags/}" + VERSION="${TAG#v}" + PRERELEASE_FLAG="" + if [[ "$VERSION" == *rc* || "$VERSION" == *alpha* || "$VERSION" == *beta* ]]; then + PRERELEASE_FLAG="--prerelease" fi - - # ========================================== - # Post-Release Validation - # ========================================== - post-release-validation: - name: Post-Release Validation - runs-on: ubuntu-latest - needs: [validate-release, create-github-release, publish-pypi, update-docs] - if: always() - steps: - - name: Validate release completion - run: | - VERSION="${{ needs.validate-release.outputs.version }}" - TAG="${{ needs.validate-release.outputs.tag }}" - - echo "๐Ÿ” Validating release completion for $TAG" - - # Check GitHub release - if gh release view "$TAG" >/dev/null 2>&1; then - echo "โœ… GitHub release exists" - else - echo "โŒ GitHub release missing" - exit 1 - fi - - # Check PyPI (skip for prereleases) - if [[ "${{ needs.validate-release.outputs.is-prerelease }}" == "false" ]]; then - if pip index versions markitdown-mcp | grep -q "$VERSION"; then - echo "โœ… PyPI package published" - else - echo "โŒ PyPI package missing" - exit 1 - fi - fi - - echo "๐ŸŽ‰ Release validation completed successfully!" - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - name: Notify on failure - if: failure() - run: | - echo "๐Ÿšจ Release workflow failed for ${{ needs.validate-release.outputs.tag }}" - echo "Manual intervention may be required" - # In a real setup, this could send notifications to Slack/email/etc. + gh release create "$TAG" dist/* \ + --title "Release $VERSION" \ + --generate-notes \ + --verify-tag \ + $PRERELEASE_FLAG diff --git a/CHANGELOG.md b/CHANGELOG.md index 62f281c..72bfade 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,22 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased] + +### Changed +- **Renamed the PyPI distribution to `trsdn-markitdown-mcp`** (the name `markitdown-mcp` + is taken by Microsoft's official project). The import package (`markitdown_mcp`) and the + `markitdown-mcp` console command are unchanged. +- Release workflow now publishes via PyPI Trusted Publishing (OIDC, `environment: pypi`) + instead of API tokens. +- `MANIFEST.in` prunes tests, docs, examples, scripts and schemas from the published + sdist. + +### Added +- `trsdn-markitdown-mcp` console-script alias and `python -m markitdown_mcp` support. +- Packaging metadata: SPDX `license`, `license-files`, maintainers, extra classifiers, + keywords and project URLs. + ## [1.2.2] - 2026-06-10 ### Fixed diff --git a/MANIFEST.in b/MANIFEST.in index 8c346be..33e876e 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,7 +1,22 @@ include README.md +include LICENSE +include CHANGELOG.md include requirements.txt -include mcp_config.json recursive-include markitdown_mcp *.py + +# Keep development-only assets out of the published distributions +prune tests +prune docs +prune examples +prune scripts +prune schemas +prune .github +exclude .env +exclude .env.example +exclude pytest.ini +exclude requirements-all.txt +exclude AGENTS.md +exclude RELEASE.md global-exclude *.pyc global-exclude __pycache__ -global-exclude .DS_Store \ No newline at end of file +global-exclude .DS_Store diff --git a/README.md b/README.md index a55e112..d9a5413 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,7 @@ # ๐Ÿ“„ MarkItDown MCP Server [![MCP](https://img.shields.io/badge/Model_Context_Protocol-MCP-blue)](https://modelcontextprotocol.io) +[![PyPI](https://img.shields.io/pypi/v/trsdn-markitdown-mcp.svg)](https://pypi.org/project/trsdn-markitdown-mcp/) [![Python](https://img.shields.io/badge/python-3.10+-blue.svg)](https://python.org) [![License](https://img.shields.io/badge/license-MIT-green.svg)](LICENSE) [![CI](https://github.com/trsdn/markitdown-mcp/workflows/CI/badge.svg)](https://github.com/trsdn/markitdown-mcp/actions) @@ -8,6 +9,19 @@ A powerful **Model Context Protocol (MCP) server** that converts 29+ file formats to clean, structured Markdown using Microsoft's MarkItDown library. +> [!IMPORTANT] +> **This is not Microsoft's official `markitdown-mcp` package.** +> This is an independent community project published on PyPI as +> **[`trsdn-markitdown-mcp`](https://pypi.org/project/trsdn-markitdown-mcp/)**. +> It *uses* Microsoft's [`markitdown`](https://pypi.org/project/markitdown/) library as a +> dependency, but it is developed and maintained separately from +> [`markitdown-mcp`](https://pypi.org/project/markitdown-mcp/) by Microsoft. +> +> - **PyPI distribution name**: `trsdn-markitdown-mcp` +> - **Python import name**: `markitdown_mcp` +> - **CLI command**: `markitdown-mcp` (alias: `trsdn-markitdown-mcp`) + + ๐Ÿ”ฅ **Perfect for Claude Desktop, MCP clients, and AI workflows!** ## โœจ Features @@ -23,11 +37,11 @@ A powerful **Model Context Protocol (MCP) server** that converts 29+ file format | File Type | Required Dependencies | Install Command | |-----------|----------------------|-----------------| -| **PDF** | `pypdf`, `pymupdf`, `pdfplumber` | `pipx inject markitdown-mcp 'markitdown[all]'` | -| **Excel (.xlsx, .xls)** | `openpyxl`, `xlrd`, `pandas` | `pipx inject markitdown-mcp openpyxl xlrd pandas` | +| **PDF** | `pypdf`, `pymupdf`, `pdfplumber` | `pipx inject trsdn-markitdown-mcp 'markitdown[all]'` | +| **Excel (.xlsx, .xls)** | `openpyxl`, `xlrd`, `pandas` | `pipx inject trsdn-markitdown-mcp openpyxl xlrd pandas` | | **PowerPoint (.pptx)** | `python-pptx` | Included in base install | | **Images** | `PIL`, `exiftool` (optional) | Included in base install | -| **Audio** | `pydub`, `speech_recognition` | `pipx inject markitdown-mcp 'markitdown[all]'` | +| **Audio** | `pydub`, `speech_recognition` | `pipx inject trsdn-markitdown-mcp 'markitdown[all]'` | | **Basic formats** | None | Base install only | **Note**: For the best experience, we recommend installing all dependencies using the **Complete Install** method below. @@ -41,8 +55,8 @@ A powerful **Model Context Protocol (MCP) server** that converts 29+ file format 1. **Install the server with ALL features:** ```bash # One command to install everything - pipx install git+https://github.com/trsdn/markitdown-mcp.git && \ - pipx inject markitdown-mcp 'markitdown[all]' openpyxl xlrd pandas pymupdf pdfplumber + pipx install trsdn-markitdown-mcp && \ + pipx inject trsdn-markitdown-mcp 'markitdown[all]' openpyxl xlrd pandas pymupdf pdfplumber ``` 2. **Add to your Claude Desktop config:** @@ -114,22 +128,34 @@ Convert all supported files in a directory. ## Installation -### Option 1: Pip Install (Recommended) +> Published on PyPI as **`trsdn-markitdown-mcp`** โ€” not to be confused with Microsoft's `markitdown-mcp`. + +### Option 1: Install from PyPI (Recommended) ```bash -# Install from local directory -pip install -e /Users/torstenmahr/GitHub/markitdown-mcp +# Isolated install with pipx +pipx install trsdn-markitdown-mcp + +# Or with pip +pip install trsdn-markitdown-mcp + +# Or run without installing (uv) +uvx trsdn-markitdown-mcp +``` -# Or navigate to the directory first -cd /Users/torstenmahr/GitHub/markitdown-mcp -pip install -e . +### Option 2: Install from source (development) + +```bash +git clone https://github.com/trsdn/markitdown-mcp.git +cd markitdown-mcp +pip install -e ".[all]" ``` -### Option 2: Direct Usage +### Option 3: Direct usage from a checkout ```bash -cd /Users/torstenmahr/GitHub/markitdown-mcp -source venv/bin/activate +cd markitdown-mcp +python -m venv venv && source venv/bin/activate pip install -r requirements.txt ``` @@ -137,15 +163,14 @@ pip install -r requirements.txt ### MCP Server Mode (Recommended) -After pip installation: +After installation: ```bash # Start the MCP server (for use with MCP clients) markitdown-mcp -``` -Or using the development script: -```bash -python run_server.py +# Equivalent alternatives +trsdn-markitdown-mcp +python -m markitdown_mcp ``` ## ๐Ÿ› ๏ธ Installation Options @@ -154,8 +179,8 @@ python run_server.py Install with ALL dependencies in one command: ```bash # Using pipx (recommended) -pipx install git+https://github.com/trsdn/markitdown-mcp.git && \ -pipx inject markitdown-mcp 'markitdown[all]' openpyxl xlrd pandas pymupdf pdfplumber pytesseract pydub speechrecognition +pipx install trsdn-markitdown-mcp && \ +pipx inject trsdn-markitdown-mcp 'markitdown[all]' openpyxl xlrd pandas pymupdf pdfplumber pytesseract pydub speechrecognition # Or download and run the install script curl -sSL https://raw.githubusercontent.com/trsdn/markitdown-mcp/main/scripts/install-all-deps.sh | bash @@ -163,7 +188,7 @@ curl -sSL https://raw.githubusercontent.com/trsdn/markitdown-mcp/main/scripts/in ### Quick Install (Basic Features Only) ```bash -pip install -e git+https://github.com/trsdn/markitdown-mcp.git +pip install trsdn-markitdown-mcp ``` ### Complete Install with All Dependencies (Step by Step) @@ -173,12 +198,12 @@ To ensure all file formats are supported, use one of these methods: #### Method 1: Using pipx (Recommended) ```bash # Install the MCP server -pipx install git+https://github.com/trsdn/markitdown-mcp.git +pipx install trsdn-markitdown-mcp # Install all required dependencies for full functionality -pipx inject markitdown-mcp 'markitdown[all]' # PDF, OCR, Speech -pipx inject markitdown-mcp openpyxl xlrd pandas # Excel support -pipx inject markitdown-mcp pymupdf pdfplumber # Advanced PDF +pipx inject trsdn-markitdown-mcp 'markitdown[all]' # PDF, OCR, Speech +pipx inject trsdn-markitdown-mcp openpyxl xlrd pandas # Excel support +pipx inject trsdn-markitdown-mcp pymupdf pdfplumber # Advanced PDF ``` #### Method 2: Using pip with virtual environment @@ -200,14 +225,14 @@ If you already have the MCP server installed but some formats aren't working: which markitdown-mcp # Shows path like /Users/you/.local/bin/markitdown-mcp # Inject missing dependencies -pipx inject markitdown-mcp 'markitdown[all]' openpyxl xlrd pandas pymupdf pdfplumber +pipx inject trsdn-markitdown-mcp 'markitdown[all]' openpyxl xlrd pandas pymupdf pdfplumber ``` ### Verify Installation -After installation, verify all dependencies are properly installed: +After installation, verify the server responds to MCP requests: ```bash -# Test the MCP server -markitdown-mcp --help +# Ask the server for its tool list over stdio (JSON-RPC) +echo '{"jsonrpc":"2.0","id":1,"method":"tools/list","params":{}}' | markitdown-mcp # For pipx installations, check injected packages pipx list --include-injected @@ -228,6 +253,20 @@ Add this to your Claude Desktop `claude_desktop_config.json`: } ``` +Prefer running it without a permanent install? Use `uvx` (the package name is +`trsdn-markitdown-mcp`, the command is `markitdown-mcp`): + +```json +{ + "mcpServers": { + "markitdown": { + "command": "uvx", + "args": ["trsdn-markitdown-mcp"] + } + } +} +``` + **Config file locations:** - **macOS**: `~/Library/Application Support/Claude/claude_desktop_config.json` - **Windows**: `%APPDATA%\Claude\claude_desktop_config.json` @@ -326,7 +365,6 @@ Convert all supported files in a directory. markitdown-mcp/ โ”œโ”€โ”€ mcp_server.py # MCP protocol server โ”œโ”€โ”€ mdconvert.py # CLI script -โ”œโ”€โ”€ run_server.py # Server runner script โ”œโ”€โ”€ mcp_config.json # MCP configuration โ”œโ”€โ”€ requirements.txt # Python dependencies โ”œโ”€โ”€ README.md # This file diff --git a/RELEASE.md b/RELEASE.md index 3e1b564..bfe8506 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -1,6 +1,11 @@ # Release Process -This document outlines the release process for markitdown-mcp. +This document outlines the release process for `trsdn-markitdown-mcp`. + +> **Naming**: the PyPI distribution is `trsdn-markitdown-mcp`. The Python import package +> stays `markitdown_mcp` and the CLI command stays `markitdown-mcp` (with a +> `trsdn-markitdown-mcp` alias). This project is **not** Microsoft's official +> `markitdown-mcp` package. ## Version Numbering @@ -20,13 +25,38 @@ We follow [Semantic Versioning](https://semver.org/): - Include all changes, fixes, and new features 3. **Create Release** - - Tag the release: `git tag -a v1.0.0 -m "Release v1.0.0"` - - Push tags: `git push origin --tags` + - Tag the release: `git tag -a v1.2.3 -m "Release v1.2.3"` + - Push tags: `git push origin v1.2.3` + +4. **Automated Publish** + - `.github/workflows/release.yml` builds the sdist + wheel, runs `twine check`, + smoke-tests the wheel, and publishes to PyPI. + +## PyPI Trusted Publishing (OIDC) + +Releases are published with [PyPI Trusted Publishing](https://docs.pypi.org/trusted-publishers/). +**No API tokens or secrets are stored in this repository.** + +The PyPI publisher configuration must match exactly: + +| Setting | Value | +|---------|-------| +| PyPI project | `trsdn-markitdown-mcp` | +| Owner / repository | `trsdn/markitdown-mcp` | +| Workflow file | `release.yml` | +| Environment | `pypi` | + +The `publish` job therefore declares `environment: pypi` and +`permissions: { id-token: write }`. Changing the workflow filename, the job's +environment, or the repository slug will break OIDC authentication. -4. **Publish to PyPI** - - Build: `python -m build` - - Upload: `python -m twine upload dist/*` +### Local dry run -## Automated Releases +```bash +python -m pip install --upgrade build twine +python -m build +twine check dist/* +``` -GitHub Actions automatically publishes to PyPI when a new tag is pushed. \ No newline at end of file +Never run `twine upload` manually โ€” publishing happens only through the tagged +release workflow. diff --git a/docs/guides/KNOWN_ISSUES.md b/docs/guides/KNOWN_ISSUES.md index 93d49eb..15162a1 100644 --- a/docs/guides/KNOWN_ISSUES.md +++ b/docs/guides/KNOWN_ISSUES.md @@ -35,7 +35,7 @@ PptxConverter threw BadZipFile... **Solution**: Install the required dependencies using pipx: ```bash -pipx inject markitdown-mcp openpyxl xlrd pandas tabulate +pipx inject trsdn-markitdown-mcp openpyxl xlrd pandas tabulate ``` ## Corrupted Office Files diff --git a/docs/index.rst b/docs/index.rst index ce3b938..b3ccf4a 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -83,11 +83,11 @@ Installation Install via pip:: - pip install markitdown-mcp + pip install trsdn-markitdown-mcp Or with all optional dependencies:: - pip install markitdown-mcp[all] + pip install trsdn-markitdown-mcp[all] Quick Example ------------- diff --git a/markitdown_mcp/__main__.py b/markitdown_mcp/__main__.py new file mode 100644 index 0000000..be5dfcd --- /dev/null +++ b/markitdown_mcp/__main__.py @@ -0,0 +1,6 @@ +"""Allow running the MCP server via ``python -m markitdown_mcp``.""" + +from .server import main + +if __name__ == "__main__": + main() diff --git a/pyproject.toml b/pyproject.toml index 7a0e62a..7e169d6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,29 +1,39 @@ [build-system] requires = [ - "setuptools>=61.0", + "setuptools>=77.0", "wheel", ] build-backend = "setuptools.build_meta" [project] -name = "markitdown-mcp" +name = "trsdn-markitdown-mcp" version = "1.2.2" -description = "A Model Context Protocol server for converting documents to Markdown using MarkItDown" +description = "An independent Model Context Protocol (MCP) server for converting documents to Markdown using MarkItDown" authors = [ - { name = "MarkItDown MCP", email = "noreply@example.com" }, + { name = "trsdn", email = "noreply@users.noreply.github.com" }, +] +maintainers = [ + { name = "trsdn", email = "noreply@users.noreply.github.com" }, ] readme = "README.md" +license = "MIT" +license-files = [ + "LICENSE", +] requires-python = ">=3.10" classifiers = [ "Development Status :: 4 - Beta", + "Environment :: Console", "Intended Audience :: Developers", - "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", "Topic :: Text Processing :: Markup :: Markdown", "Topic :: Software Development :: Libraries :: Python Modules", + "Typing :: Typed", ] keywords = [ "mcp", @@ -31,6 +41,10 @@ keywords = [ "markdown", "document-conversion", "model-context-protocol", + "pdf", + "docx", + "ocr", + "llm", ] dependencies = [ "markitdown>=0.1.0", @@ -42,9 +56,6 @@ dependencies = [ "python-dotenv>=1.0.0", ] -[project.license] -text = "MIT" - [project.optional-dependencies] all = [ "markitdown[all]>=0.1.0", @@ -92,11 +103,14 @@ docs = [ [project.urls] Homepage = "https://github.com/trsdn/markitdown-mcp" +Documentation = "https://github.com/trsdn/markitdown-mcp#readme" Repository = "https://github.com/trsdn/markitdown-mcp" Issues = "https://github.com/trsdn/markitdown-mcp/issues" +Changelog = "https://github.com/trsdn/markitdown-mcp/blob/main/CHANGELOG.md" [project.scripts] markitdown-mcp = "markitdown_mcp.server:main" +trsdn-markitdown-mcp = "markitdown_mcp.server:main" [tool.setuptools.packages.find] where = [ diff --git a/scripts/install-all-deps.sh b/scripts/install-all-deps.sh index 75c86a1..d4b3998 100755 --- a/scripts/install-all-deps.sh +++ b/scripts/install-all-deps.sh @@ -17,7 +17,7 @@ fi # Install or reinstall the MCP server echo "๐Ÿ“ฆ Installing MarkItDown MCP Server..." -pipx install --force git+https://github.com/trsdn/markitdown-mcp.git +pipx install --force trsdn-markitdown-mcp # Install all required dependencies echo "" @@ -25,19 +25,19 @@ echo "๐Ÿ’‰ Injecting all required dependencies..." # Core MarkItDown dependencies echo " โ†’ Installing PDF, OCR, and Speech dependencies..." -pipx inject markitdown-mcp 'markitdown[all]' --force +pipx inject trsdn-markitdown-mcp 'markitdown[all]' --force # Excel support echo " โ†’ Installing Excel support..." -pipx inject markitdown-mcp openpyxl xlrd pandas tabulate --force +pipx inject trsdn-markitdown-mcp openpyxl xlrd pandas tabulate --force # Advanced PDF support echo " โ†’ Installing advanced PDF support..." -pipx inject markitdown-mcp pymupdf pdfplumber pytesseract --force +pipx inject trsdn-markitdown-mcp pymupdf pdfplumber pytesseract --force # Audio/video support echo " โ†’ Installing audio processing support..." -pipx inject markitdown-mcp pydub speechrecognition --force +pipx inject trsdn-markitdown-mcp pydub speechrecognition --force echo "" echo "โœ… Installation complete!" From eb0091c19ac018841ce376edb270fd0e995d86c2 Mon Sep 17 00:00:00 2001 From: Torsten Mahr Date: Mon, 24 Aug 2026 23:32:18 +0200 Subject: [PATCH 2/3] fix(ci): restore full release pipeline and add publish job only Reverts the over-aggressive rewrite of release.yml. The original job chain (validate-release, quality-gates with the 3.10/3.11/3.12 matrix, security-scan, build-package, generate-changelog, create-github-release, update-docs, post-release-validation) and the concurrency group are restored unchanged. Trusted Publishing is now additive: - new `publish` job with needs: [validate-release, quality-gates, security-scan, build-package, generate-changelog], `environment: pypi`, `permissions: { id-token: write }` and pypa/gh-action-pypi-publish, no secrets - `create-github-release` now needs `publish`, so the GitHub release is created after the PyPI upload (and still runs for prereleases, where publish is skipped) - workflow-level permissions reduced to `contents: read`; `id-token: write` only in `publish`, `contents: write` only in `create-github-release`/`update-docs` - `workflow_dispatch` with a required `tag` input; all checkouts resolve `inputs.tag || github.ref` so a manual run builds the tagged commit - version sync no longer uses a bare `sed s/version = ".*"/`, which also rewrote `target-version = "py310"` and `python_version = "3.10"` in the [tool.*] sections of pyproject.toml - package verification now exercises markitdown-mcp, trsdn-markitdown-mcp and `python -m markitdown_mcp` with a real tools/list request, and reads the version via importlib.metadata; PyPI names updated to trsdn-markitdown-mcp Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/workflows/release.yml | 566 ++++++++++++++++++++++++++++++---- RELEASE.md | 15 +- 2 files changed, 519 insertions(+), 62 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 8e349a0..8dd5061 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -3,9 +3,16 @@ name: Release on: push: tags: - - 'v*' + - 'v*.*.*' workflow_dispatch: + inputs: + tag: + description: 'Existing release tag to (re-)run the pipeline for, e.g. v1.2.3' + required: true + type: string +# Least privilege by default; jobs opt into what they need. +# `id-token: write` is granted ONLY to the publish job (PyPI Trusted Publishing). permissions: contents: read @@ -16,32 +23,189 @@ concurrency: jobs: # ========================================== - # Build sdist + wheel + # Pre-Release Validation # ========================================== - build: - name: Build distributions + validate-release: + name: Validate Release runs-on: ubuntu-latest + outputs: + version: ${{ steps.version.outputs.version }} + tag: ${{ steps.version.outputs.tag }} + is-prerelease: ${{ steps.version.outputs.prerelease }} steps: - uses: actions/checkout@v4 with: fetch-depth: 0 + ref: ${{ inputs.tag || github.ref }} + + - name: Extract version from tag + id: version + run: | + TAG="${{ inputs.tag || github.ref_name }}" + VERSION=${TAG#v} + + echo "tag=${TAG}" >> $GITHUB_OUTPUT + echo "version=${VERSION}" >> $GITHUB_OUTPUT + + # Check if this is a prerelease (contains rc, alpha, beta) + if [[ $VERSION == *"rc"* ]] || [[ $VERSION == *"alpha"* ]] || [[ $VERSION == *"beta"* ]]; then + echo "prerelease=true" >> $GITHUB_OUTPUT + else + echo "prerelease=false" >> $GITHUB_OUTPUT + fi + + - name: Validate tag format + run: | + TAG="${{ steps.version.outputs.tag }}" + if [[ ! $TAG =~ ^v[0-9]+\.[0-9]+\.[0-9]+(-.*)?$ ]]; then + echo "โŒ Invalid tag format: $TAG" + echo "Expected format: v1.2.3 or v1.2.3-rc.1" + exit 1 + fi + echo "โœ… Valid tag format: $TAG" + + - name: Check if tag already exists in releases + run: | + TAG="${{ steps.version.outputs.tag }}" + if gh release view "$TAG" >/dev/null 2>&1; then + echo "โŒ Release for tag $TAG already exists" + exit 1 + fi + echo "โœ… New release tag: $TAG" + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + # ========================================== + # Quality Gates - Must Pass + # ========================================== + quality-gates: + name: Release Quality Gates + runs-on: ubuntu-latest + needs: validate-release + strategy: + matrix: + python-version: ['3.10', '3.11', '3.12'] + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ inputs.tag || github.ref }} + + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + cache: 'pip' + + - name: Install dependencies + run: | + pip install --upgrade pip + pip install -e ".[dev]" + + - name: Format check + run: ruff format --check + + - name: Lint check + run: ruff check + + - name: Type check + run: mypy markitdown_mcp/ --strict + + - name: Run tests with coverage + run: | + pytest --cov=markitdown_mcp --cov-report=json --cov-fail-under=80 \ + tests/unit/ tests/integration/test_mcp_protocol_smoke.py + + - name: MCP protocol validation + run: | + python scripts/generate-schemas.py + + # Verify MCP server starts and responds + timeout 30 python -c " + import asyncio + from markitdown_mcp.server import MarkItDownMCPServer, MCPRequest + + async def test(): + server = MarkItDownMCPServer() + req = MCPRequest(id='test', method='tools/list', params={}) + resp = await server.handle_request(req) + assert resp.result, 'MCP server failed to respond' + tools = resp.result['tools'] + assert len(tools) >= 3, f'Expected 3+ tools, got {len(tools)}' + print(f'โœ… MCP validation passed: {len(tools)} tools available') + + asyncio.run(test()) + " + + # ========================================== + # Security Validation + # ========================================== + security-scan: + name: Security Validation + runs-on: ubuntu-latest + needs: validate-release + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ inputs.tag || github.ref }} - uses: actions/setup-python@v5 with: python-version: '3.12' - - name: Install build tooling - run: python -m pip install --upgrade pip build twine + - name: Install security tools + run: | + pip install bandit safety - - name: Sync version with tag - if: startsWith(github.ref, 'refs/tags/v') + - name: Security scan with Bandit run: | - TAG="${GITHUB_REF#refs/tags/}" - VERSION="${TAG#v}" - if [[ ! "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+([.-].*)?$ ]]; then - echo "::error::Invalid tag format: $TAG (expected vX.Y.Z)" - exit 1 - fi + bandit -r markitdown_mcp/ -f json -o bandit-results.json + python -c " + import json + with open('bandit-results.json') as f: + results = json.load(f) + high_issues = [r for r in results.get('results', []) if r['issue_severity'] == 'HIGH'] + if high_issues: + print(f'โŒ Found {len(high_issues)} HIGH severity security issues') + for issue in high_issues[:3]: + print(f' - {issue[\"test_name\"]}: {issue[\"issue_text\"]}') + exit(1) + print('โœ… No high-severity security issues found') + " + + - name: Dependency vulnerability scan + run: | + safety check --json || true + + # ========================================== + # Build and Test Package + # ========================================== + build-package: + name: Build Package + runs-on: ubuntu-latest + needs: [validate-release, quality-gates, security-scan] + outputs: + package-name: ${{ steps.build.outputs.package-name }} + package-version: ${{ steps.build.outputs.package-version }} + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ inputs.tag || github.ref }} + + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Install build tools + run: | + pip install build twine tomli_w + + - name: Build package + id: build + run: | + # Update version in pyproject.toml to match tag. + # NOTE: a plain `sed s/version = ".*"/.../` also rewrites `python_version` + # and `target-version` in the [tool.*] sections, so anchor on the line start + # and replace exactly one occurrence. + VERSION="${{ needs.validate-release.outputs.version }}" python - "$VERSION" <<'PY' import pathlib import re @@ -54,61 +218,224 @@ jobs: r'^version = ".*"$', f'version = "{version}"', text, count=1, flags=re.M ) if count != 1: - raise SystemExit("Could not update version in pyproject.toml") + raise SystemExit("Could not update [project] version in pyproject.toml") path.write_text(text, encoding="utf-8") - print(f"pyproject.toml version set to {version}") PY - - name: Build sdist and wheel - run: python -m build + # Build package + python -m build + + # Extract package info + PACKAGE_NAME=$(python -c "import tomllib; print(tomllib.load(open('pyproject.toml', 'rb'))['project']['name'])") + echo "package-name=$PACKAGE_NAME" >> $GITHUB_OUTPUT + echo "package-version=$VERSION" >> $GITHUB_OUTPUT + + echo "๐Ÿ“ฆ Built package: $PACKAGE_NAME v$VERSION" - - name: Check distributions + - name: Verify package run: | + # Check package contents twine check dist/* - echo "### Built artifacts" >> "$GITHUB_STEP_SUMMARY" - find dist -type f -printf '%f (%s bytes)\n' | tee -a "$GITHUB_STEP_SUMMARY" - - name: Smoke test wheel - run: | - python -m venv /tmp/smoke - /tmp/smoke/bin/pip install --upgrade pip - /tmp/smoke/bin/pip install dist/*.whl - /tmp/smoke/bin/python -c "import markitdown_mcp; print('import ok')" + # Install and test package + pip install dist/*.whl + + # Test basic functionality + python -c " + from importlib.metadata import version + + import markitdown_mcp + + print(f'โœ… Package installed successfully: {version(\"trsdn-markitdown-mcp\")}') + print(f' import package: {markitdown_mcp.__name__}') + " + + # Test every documented entrypoint against a real MCP request REQUEST='{"jsonrpc":"2.0","id":1,"method":"tools/list","params":{}}' - echo "$REQUEST" | timeout 60 /tmp/smoke/bin/markitdown-mcp | tee /tmp/console-script.json - echo "$REQUEST" | timeout 60 /tmp/smoke/bin/trsdn-markitdown-mcp > /dev/null - echo "$REQUEST" | timeout 60 /tmp/smoke/bin/python -m markitdown_mcp > /dev/null - /tmp/smoke/bin/python -c " - import json, pathlib, sys - data = json.loads(pathlib.Path('/tmp/console-script.json').read_text()) - tools = data['result']['tools'] + echo "$REQUEST" | timeout 60 markitdown-mcp | tee /tmp/tools-list.json + echo "$REQUEST" | timeout 60 trsdn-markitdown-mcp > /dev/null + echo "$REQUEST" | timeout 60 python -m markitdown_mcp > /dev/null + + python -c " + import json + import pathlib + + tools = json.loads(pathlib.Path('/tmp/tools-list.json').read_text())['result']['tools'] assert len(tools) >= 3, f'expected 3+ tools, got {len(tools)}' - print(f'MCP entrypoint OK: {len(tools)} tools') + print(f'โœ… MCP entrypoints OK: {len(tools)} tools') " - - name: Upload distributions + - name: Upload build artifacts uses: actions/upload-artifact@v4 with: - name: dist + name: dist-${{ needs.validate-release.outputs.version }} path: dist/ if-no-files-found: error retention-days: 30 # ========================================== - # Publish to PyPI via Trusted Publishing (OIDC) + # Generate Changelog # ========================================== + generate-changelog: + name: Generate Changelog + runs-on: ubuntu-latest + needs: validate-release + outputs: + changelog: ${{ steps.changelog.outputs.changelog }} + release-notes: ${{ steps.changelog.outputs.release-notes }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{ inputs.tag || github.ref }} + + - name: Generate changelog from commits + id: changelog + run: | + # Get previous tag for changelog generation + CURRENT_TAG="${{ needs.validate-release.outputs.tag }}" + PREVIOUS_TAG=$(git describe --tags --abbrev=0 $CURRENT_TAG^ 2>/dev/null || echo "") + + if [ -z "$PREVIOUS_TAG" ]; then + echo "First release - using all commits" + COMMIT_RANGE="HEAD" + else + echo "Previous tag: $PREVIOUS_TAG" + COMMIT_RANGE="$PREVIOUS_TAG..$CURRENT_TAG" + fi + + # Generate changelog from conventional commits + python -c " + import subprocess + import re + from datetime import datetime + + # Get commits in range + result = subprocess.run(['git', 'log', '--oneline', '--pretty=format:%s', '$COMMIT_RANGE'], + capture_output=True, text=True) + commits = result.stdout.strip().split('\n') if result.stdout.strip() else [] + + # Parse conventional commits + features = [] + fixes = [] + breaking = [] + other = [] + + for commit in commits: + if not commit.strip(): + continue + + # Check for breaking changes + if 'BREAKING CHANGE' in commit.upper(): + breaking.append(commit) + elif commit.startswith('feat'): + features.append(commit) + elif commit.startswith('fix'): + fixes.append(commit) + elif commit.startswith(('docs', 'test', 'chore', 'ci', 'refactor', 'perf')): + other.append(commit) + + # Generate changelog + changelog = [] + changelog.append(f'## [{\"${{ needs.validate-release.outputs.version }}\"}] - {datetime.now().strftime(\"%Y-%m-%d\")}') + changelog.append('') + + if breaking: + changelog.append('### โš ๏ธ BREAKING CHANGES') + for commit in breaking: + desc = re.sub(r'^[^:]+:', '', commit).strip() + changelog.append(f'- {desc}') + changelog.append('') + + if features: + changelog.append('### โœจ Features') + for commit in features: + desc = re.sub(r'^feat[^:]*:', '', commit).strip() + changelog.append(f'- {desc}') + changelog.append('') + + if fixes: + changelog.append('### ๐Ÿ› Bug Fixes') + for commit in fixes: + desc = re.sub(r'^fix[^:]*:', '', commit).strip() + changelog.append(f'- {desc}') + changelog.append('') + + if other: + changelog.append('### ๐Ÿ”ง Other Changes') + for commit in other[:5]: # Limit other changes + desc = re.sub(r'^[^:]+:', '', commit).strip() + changelog.append(f'- {desc}') + if len(other) > 5: + changelog.append(f'- ... and {len(other) - 5} more changes') + changelog.append('') + + changelog_text = '\n'.join(changelog) + + # Create release notes (shorter version for GitHub) + release_notes = [] + if breaking: + release_notes.append('### โš ๏ธ BREAKING CHANGES') + for commit in breaking[:3]: + desc = re.sub(r'^[^:]+:', '', commit).strip() + release_notes.append(f'- {desc}') + release_notes.append('') + + if features: + release_notes.append('### โœจ New Features') + for commit in features[:5]: + desc = re.sub(r'^feat[^:]*:', '', commit).strip() + release_notes.append(f'- {desc}') + if len(features) > 5: + release_notes.append(f'- ... and {len(features) - 5} more features') + release_notes.append('') + + if fixes: + release_notes.append('### ๐Ÿ› Bug Fixes') + for commit in fixes[:5]: + desc = re.sub(r'^fix[^:]*:', '', commit).strip() + release_notes.append(f'- {desc}') + if len(fixes) > 5: + release_notes.append(f'- ... and {len(fixes) - 5} more fixes') + release_notes.append('') + + release_notes.append('### ๐Ÿ“Š Release Info') + release_notes.append(f'- **Total commits**: {len(commits)}') + release_notes.append(f'- **Features**: {len(features)}') + release_notes.append(f'- **Bug fixes**: {len(fixes)}') + if breaking: + release_notes.append(f'- **Breaking changes**: {len(breaking)}') + + release_notes_text = '\n'.join(release_notes) + + # Output to GitHub Actions + import os + with open(os.environ['GITHUB_OUTPUT'], 'a') as f: + f.write(f'changelog< CHANGELOG.md + + # Clean up + rm CHANGELOG.md.bak + + - name: Commit changelog update + run: | + git config --local user.email "action@github.com" + git config --local user.name "GitHub Action" + + if git diff --exit-code CHANGELOG.md; then + echo "No changelog changes to commit" + else + git add CHANGELOG.md + git commit -m "docs: update changelog for v${{ needs.validate-release.outputs.version }}" + git push origin main + echo "โœ… Changelog updated" fi - gh release create "$TAG" dist/* \ - --title "Release $VERSION" \ - --generate-notes \ - --verify-tag \ - $PRERELEASE_FLAG + + # ========================================== + # Post-Release Validation + # ========================================== + post-release-validation: + name: Post-Release Validation + runs-on: ubuntu-latest + needs: [validate-release, create-github-release, publish, update-docs] + if: always() + steps: + - name: Validate release completion + run: | + VERSION="${{ needs.validate-release.outputs.version }}" + TAG="${{ needs.validate-release.outputs.tag }}" + + echo "๐Ÿ” Validating release completion for $TAG" + + # Check GitHub release + if gh release view "$TAG" >/dev/null 2>&1; then + echo "โœ… GitHub release exists" + else + echo "โŒ GitHub release missing" + exit 1 + fi + + # Check PyPI (skip for prereleases) + if [[ "${{ needs.validate-release.outputs.is-prerelease }}" == "false" ]]; then + if pip index versions trsdn-markitdown-mcp | grep -q "$VERSION"; then + echo "โœ… PyPI package published" + else + echo "โŒ PyPI package missing" + exit 1 + fi + fi + + echo "๐ŸŽ‰ Release validation completed successfully!" + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Notify on failure + if: failure() + run: | + echo "๐Ÿšจ Release workflow failed for ${{ needs.validate-release.outputs.tag }}" + echo "Manual intervention may be required" + # In a real setup, this could send notifications to Slack/email/etc. diff --git a/RELEASE.md b/RELEASE.md index bfe8506..189245e 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -29,8 +29,13 @@ We follow [Semantic Versioning](https://semver.org/): - Push tags: `git push origin v1.2.3` 4. **Automated Publish** - - `.github/workflows/release.yml` builds the sdist + wheel, runs `twine check`, - smoke-tests the wheel, and publishes to PyPI. + - `.github/workflows/release.yml` runs the full release chain: + `validate-release` โ†’ `quality-gates` (ruff/mypy/pytest on Python 3.10/3.11/3.12 + plus MCP protocol validation) + `security-scan` (Bandit, dependency audit) โ†’ + `build-package` (build, `twine check`, entrypoint smoke test) โ†’ + `generate-changelog` โ†’ `publish` (PyPI) โ†’ `create-github-release` โ†’ + `update-docs` โ†’ `post-release-validation`. + - The GitHub release is created **after** the PyPI publish succeeds. ## PyPI Trusted Publishing (OIDC) @@ -47,8 +52,10 @@ The PyPI publisher configuration must match exactly: | Environment | `pypi` | The `publish` job therefore declares `environment: pypi` and -`permissions: { id-token: write }`. Changing the workflow filename, the job's -environment, or the repository slug will break OIDC authentication. +`permissions: { id-token: write }`. `id-token: write` is granted **only** to that job; +the workflow-level default is `contents: read`, and `contents: write` is scoped to the +jobs that create the release and push the changelog. Changing the workflow filename, the +job's environment, or the repository slug will break OIDC authentication. ### Local dry run From 8ddd217b48d8aaed0c89b625dad4786d05554f48 Mon Sep 17 00:00:00 2001 From: Torsten Mahr Date: Mon, 24 Aug 2026 23:44:33 +0200 Subject: [PATCH 3/3] fix(ci): unbreak code-quality gates after ruff output change Both failures are pre-existing on main and unrelated to the packaging change; they only surfaced now because CI installs an unpinned `ruff>=0.1.0` and a newer release changed behaviour. - ruff now also formats Python blocks embedded in Markdown, so AGENTS.md (not touched by this PR) failed `ruff format --check`. Applied `ruff format`; the diff is purely quote/wrapping style inside documentation snippets. - pr-feedback.yml counted format issues with `$(... | grep -c "would reformat" || echo "0")`. `grep -c` prints `0` *and* exits 1 when nothing matches, so the substitution became the literal "0\n0", which broke `$((total_issues + format_issues))` and the later `[ -gt ]` comparison ("Invalid format '0'"). Newer ruff also reports "N files would be reformatted", which the old pattern never matched. Now the summary line is parsed with a safe fallback. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/workflows/pr-feedback.yml | 7 ++- AGENTS.md | 97 ++++++++++++++----------------- 2 files changed, 50 insertions(+), 54 deletions(-) diff --git a/.github/workflows/pr-feedback.yml b/.github/workflows/pr-feedback.yml index fad38ed..9dbd3cc 100644 --- a/.github/workflows/pr-feedback.yml +++ b/.github/workflows/pr-feedback.yml @@ -64,7 +64,12 @@ jobs: # =================== echo "### ๐ŸŽจ Code Formatting" >> pr-feedback.md if ! ruff format --check . > format_output.txt 2>&1; then - format_issues=$(ruff format --check . 2>&1 | grep -c "would reformat" || echo "0") + # `grep -c` prints 0 AND exits 1 when nothing matches, so `|| echo "0"` + # produced the literal string "0\n0" and broke the arithmetic below. + # Recent ruff versions also report "N files would be reformatted", so + # parse the summary line instead of counting per-file markers. + format_issues=$(grep -oE '[0-9]+ files? would be reformatted' format_output.txt | grep -oE '^[0-9]+' | tail -1 || true) + format_issues=${format_issues:-1} echo "โŒ **$format_issues file(s) need formatting**" >> pr-feedback.md echo "" >> pr-feedback.md echo "
Click to see formatting issues" >> pr-feedback.md diff --git a/AGENTS.md b/AGENTS.md index c0df831..1da8a3f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -105,7 +105,7 @@ def analyze_document(file_path): result = convert_file(file_path=file_path) # Step 2: Extract and analyze content - markdown_content = result['content'][0]['text'] + markdown_content = result["content"][0]["text"] # Step 3: Provide analysis to user return analyze_markdown(markdown_content) @@ -115,10 +115,7 @@ def analyze_document(file_path): ```python def process_document_folder(input_dir, output_dir): # Convert entire directory - result = convert_directory( - input_directory=input_dir, - output_directory=output_dir - ) + result = convert_directory(input_directory=input_dir, output_directory=output_dir) # Report results to user return summarize_conversion_results(result) @@ -174,10 +171,10 @@ convert_file("/etc/passwd") formats = list_supported_formats() if is_supported(file_path, formats): result = convert_file(file_path) - if result.get('error'): - handle_conversion_error(result['error']) + if result.get("error"): + handle_conversion_error(result["error"]) else: - process_markdown(result['content']) + process_markdown(result["content"]) ``` ## ๐Ÿ” Troubleshooting Guide @@ -271,13 +268,10 @@ class DocumentConverter: async def convert_and_analyze(self, file_path): # Convert document - result = await self.mcp.call_tool( - "convert_file", - {"file_path": file_path} - ) + result = await self.mcp.call_tool("convert_file", {"file_path": file_path}) # Extract content - markdown = result['content'][0]['text'] + markdown = result["content"][0]["text"] # Perform analysis return self.analyze_content(markdown) @@ -341,18 +335,18 @@ def analyze_version_impact(commits_since_last_release): Returns: 'major', 'minor', 'patch', or 'none' """ - has_breaking = any('BREAKING CHANGE' in commit.body for commit in commits) - has_features = any(commit.type == 'feat' for commit in commits) - has_fixes = any(commit.type in ['fix', 'perf'] for commit in commits) + has_breaking = any("BREAKING CHANGE" in commit.body for commit in commits) + has_features = any(commit.type == "feat" for commit in commits) + has_fixes = any(commit.type in ["fix", "perf"] for commit in commits) if has_breaking: - return 'major' + return "major" elif has_features: - return 'minor' + return "minor" elif has_fixes: - return 'patch' + return "patch" else: - return 'none' + return "none" ``` ### Release Readiness Validation @@ -363,13 +357,13 @@ Before suggesting releases, AI agents should verify: def validate_release_readiness(): """Check if codebase is ready for release.""" checks = { - 'format_check': run_ruff_format(), - 'lint_check': run_ruff_lint(), - 'type_check': run_mypy(), - 'tests_pass': run_test_suite(), - 'coverage_ok': check_coverage_threshold(), - 'docs_updated': verify_documentation(), - 'security_clean': run_security_scan() + "format_check": run_ruff_format(), + "lint_check": run_ruff_lint(), + "type_check": run_mypy(), + "tests_pass": run_test_suite(), + "coverage_ok": check_coverage_threshold(), + "docs_updated": verify_documentation(), + "security_clean": run_security_scan(), } failed_checks = [name for name, passed in checks.items() if not passed] @@ -389,9 +383,9 @@ def generate_changelog_entry(commits, version): changelog = f"## [{version}] - {datetime.now().strftime('%Y-%m-%d')}\n\n" # Group commits by type - features = [c for c in commits if c.type == 'feat'] - fixes = [c for c in commits if c.type == 'fix'] - breaking = [c for c in commits if 'BREAKING CHANGE' in c.body] + features = [c for c in commits if c.type == "feat"] + fixes = [c for c in commits if c.type == "fix"] + breaking = [c for c in commits if "BREAKING CHANGE" in c.body] if breaking: changelog += "### โš ๏ธ BREAKING CHANGES\n" @@ -435,7 +429,7 @@ async def suggest_release_if_needed(): # Analyze version impact version_impact = analyze_version_impact(commits) - if version_impact == 'none': + if version_impact == "none": return "No release needed - no significant changes" # Validate readiness @@ -451,11 +445,11 @@ async def suggest_release_if_needed(): changelog = generate_changelog_entry(commits, new_version) return { - 'action': 'suggest_release', - 'version': new_version, - 'changelog': changelog, - 'commits_included': len(commits), - 'impact': version_impact + "action": "suggest_release", + "version": new_version, + "changelog": changelog, + "commits_included": len(commits), + "impact": version_impact, } ``` @@ -467,20 +461,20 @@ AI agents should communicate release information clearly: def format_release_summary(release_info): """Format release information for users.""" return f""" -๐Ÿš€ **Release Suggestion: v{release_info['version']}** +๐Ÿš€ **Release Suggestion: v{release_info["version"]}** -**Impact**: {release_info['impact'].title()} version bump -**Changes**: {release_info['commits_included']} commits included -**Type**: {'๐Ÿ”ฅ Breaking changes' if release_info['impact'] == 'major' else 'โœจ New features' if release_info['impact'] == 'minor' else '๐Ÿ› Bug fixes'} +**Impact**: {release_info["impact"].title()} version bump +**Changes**: {release_info["commits_included"]} commits included +**Type**: {"๐Ÿ”ฅ Breaking changes" if release_info["impact"] == "major" else "โœจ New features" if release_info["impact"] == "minor" else "๐Ÿ› Bug fixes"} **Changelog Preview**: -{release_info['changelog'][:500]}... +{release_info["changelog"][:500]}... **Next Steps**: 1. Review the proposed changes 2. Run final quality checks -3. Create release tag: `git tag v{release_info['version']}` -4. Push tag to trigger automated release: `git push origin v{release_info['version']}` +3. Create release tag: `git tag v{release_info["version"]}` +4. Push tag to trigger automated release: `git push origin v{release_info["version"]}` """ ``` @@ -496,7 +490,7 @@ For critical security issues, AI agents should: ```python def detect_emergency_release(commits): """Detect if commits contain critical security fixes.""" - security_keywords = ['security', 'vulnerability', 'cve', 'exploit', 'xss', 'injection'] + security_keywords = ["security", "vulnerability", "cve", "exploit", "xss", "injection"] for commit in commits: commit_text = f"{commit.description} {commit.body}".lower() @@ -670,18 +664,15 @@ def create_new_workflow(name, triggers, jobs): """Template for creating new GitHub Actions workflows.""" workflow = { - 'name': name, - 'on': triggers, - 'permissions': { - 'contents': 'read', - 'pull-requests': 'write' - }, - 'jobs': jobs + "name": name, + "on": triggers, + "permissions": {"contents": "read", "pull-requests": "write"}, + "jobs": jobs, } # Validate before writing yaml_content = yaml.dump(workflow, default_flow_style=False) - return safe_workflow_update(f'.github/workflows/{name.lower()}.yml', yaml_content) + return safe_workflow_update(f".github/workflows/{name.lower()}.yml", yaml_content) ``` **Updating existing workflow**: @@ -690,7 +681,7 @@ def update_workflow_safely(file_path, updates): """Safely update existing workflow with changes.""" # Read current workflow - with open(file_path, 'r') as f: + with open(file_path, "r") as f: workflow = yaml.safe_load(f) # Apply updates