Discover and publish DSH Topic plugins #273
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Discover and publish DSH Topic plugins | |
| on: | |
| schedule: | |
| - cron: "37 3 * * *" | |
| workflow_dispatch: | |
| inputs: | |
| mode: | |
| description: Observe the Topic, publish an observation, or review one submission | |
| required: true | |
| default: observe | |
| type: choice | |
| options: | |
| - observe | |
| - publish | |
| - submission | |
| target_repository: | |
| description: Public GitHub repository URL used only in submission mode | |
| required: false | |
| type: string | |
| source_issue: | |
| description: Originating plugin submission Issue number | |
| required: false | |
| type: string | |
| max_promotions: | |
| description: Maximum observed candidates to publish; 0 publishes all | |
| required: true | |
| default: 0 | |
| type: number | |
| permissions: | |
| actions: read | |
| contents: write | |
| issues: write | |
| pull-requests: write | |
| concurrency: | |
| group: topic-plugin-sync | |
| cancel-in-progress: false | |
| jobs: | |
| discover: | |
| runs-on: ubuntu-latest | |
| env: | |
| AUTO_PUBLISH: ${{ (github.event_name == 'workflow_dispatch' && (inputs.mode == 'publish' || inputs.mode == 'submission')) || (github.event_name != 'workflow_dispatch' && vars.AUTO_PUBLISH == 'true') }} | |
| ENRICHMENT_LIMIT: "150" | |
| LLM_BASE_URL: ${{ vars.LLM_BASE_URL }} | |
| LLM_MODEL: ${{ vars.LLM_MODEL }} | |
| MODEL_ANALYSIS_LIMIT: "10" | |
| steps: | |
| - uses: actions/checkout@v6 | |
| with: | |
| fetch-depth: 0 | |
| - name: Restore the operational candidate ledger | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: | | |
| mkdir -p data .candidate-state | |
| restored="" | |
| for run_id in $(gh run list --workflow sync-topic-plugins.yml --status success --limit 20 --json databaseId --jq '.[].databaseId'); do | |
| if gh run download "$run_id" --name topic-candidate-ledger --dir .candidate-state 2>/dev/null; then | |
| restored=$(find .candidate-state -name topic-candidates.json -print -quit) | |
| break | |
| fi | |
| done | |
| test -n "$restored" | |
| cp "$restored" data/topic-candidates.json | |
| - name: Discover and assess topic candidates | |
| id: assess | |
| env: | |
| GITHUB_TOKEN: ${{ github.token }} | |
| LLM_API_KEY: ${{ secrets.LLM_API_KEY }} | |
| RUN_MODE: ${{ github.event_name == 'workflow_dispatch' && inputs.mode || 'scheduled' }} | |
| MAX_PROMOTIONS: ${{ inputs.max_promotions || 0 }} | |
| TARGET_REPOSITORY: ${{ inputs.target_repository }} | |
| run: | | |
| if [ "$RUN_MODE" = "publish" ]; then | |
| args=(approve-observed) | |
| if [ "$MAX_PROMOTIONS" != "0" ]; then | |
| args+=(--limit "$MAX_PROMOTIONS") | |
| fi | |
| python3 scripts/sync_topic_plugins.py "${args[@]}" | |
| elif [ "$RUN_MODE" = "submission" ]; then | |
| test -n "$TARGET_REPOSITORY" | |
| python3 scripts/sync_topic_plugins.py review-target \ | |
| --repository "$TARGET_REPOSITORY" --auto-publish | |
| else | |
| args=(--analyze-model) | |
| if [ "$AUTO_PUBLISH" = "true" ]; then | |
| args+=(--auto-publish) | |
| fi | |
| python3 scripts/sync_topic_plugins.py discover "${args[@]}" | |
| fi | |
| - name: Generate the public directory and operational report | |
| id: render | |
| run: | | |
| args=(--run-id "$GITHUB_RUN_ID") | |
| if [ "$AUTO_PUBLISH" = "true" ]; then | |
| args+=(--report) | |
| fi | |
| python3 scripts/sync_topic_plugins.py render "${args[@]}" | |
| - name: Validate the generated directory | |
| id: validate | |
| run: | | |
| python3 scripts/sync_topic_plugins.py check | |
| python3 scripts/validate_directory.py | |
| python3 -m unittest discover -s tests -p "test_*.py" | |
| snapshot() { | |
| git status --porcelain -- README.md README.zh.md CHANGELOG.md data/plugins.json | |
| sha256sum README.md README.zh.md CHANGELOG.md data/plugins.json | |
| } | |
| before=$(snapshot) | |
| args=(--run-id "$GITHUB_RUN_ID") | |
| if [ "$AUTO_PUBLISH" = "true" ]; then | |
| args+=(--report) | |
| fi | |
| python3 scripts/sync_topic_plugins.py render "${args[@]}" | |
| after=$(snapshot) | |
| test "$before" = "$after" | |
| - name: Draft bilingual promotion tweets for the report | |
| id: tweets | |
| if: steps.render.outputs.report_created == 'true' | |
| env: | |
| LLM_API_KEY: ${{ secrets.LLM_API_KEY }} | |
| run: | | |
| python3 scripts/sync_topic_plugins.py draft-tweets \ | |
| --report-dir "${{ steps.render.outputs.report_dir }}" | |
| - name: Save a recoverable report source bundle | |
| if: steps.render.outputs.report_created == 'true' | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: plugin-collection-report-source-${{ github.run_id }} | |
| path: ${{ steps.render.outputs.report_dir }} | |
| if-no-files-found: error | |
| retention-days: 30 | |
| - name: Detect public directory changes | |
| id: changes | |
| run: | | |
| if [ -z "$(git status --porcelain -- README.md README.zh.md CHANGELOG.md data/plugins.json)" ]; then | |
| echo "changed=false" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "changed=true" >> "$GITHUB_OUTPUT" | |
| fi | |
| - name: Publish changes through a temporary branch | |
| if: steps.changes.outputs.changed == 'true' | |
| id: pull_request | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: | | |
| branch="automation/catalog-${GITHUB_RUN_ID}" | |
| git config user.name "github-actions[bot]" | |
| git config user.email "41898282+github-actions[bot]@users.noreply.github.com" | |
| git switch --create "$branch" | |
| git add README.md README.zh.md CHANGELOG.md data/plugins.json | |
| git commit -m "chore: refresh dsh-plugin catalog" | |
| git push origin "HEAD:$branch" | |
| body=$(mktemp) | |
| python3 scripts/sync_topic_plugins.py summary > "$body" | |
| if [ "$AUTO_PUBLISH" = "true" ]; then | |
| pr_url=$(gh pr create --base main --head "$branch" --title "chore: refresh dsh-plugin catalog" --body-file "$body") | |
| else | |
| pr_url=$(gh pr create --draft --base main --head "$branch" --title "chore: refresh dsh-plugin catalog" --body-file "$body") | |
| fi | |
| echo "number=$(gh pr view "$pr_url" --json number --jq .number)" >> "$GITHUB_OUTPUT" | |
| echo "branch=$branch" >> "$GITHUB_OUTPUT" | |
| - name: Merge a successful publication and remove its temporary branch | |
| if: env.AUTO_PUBLISH == 'true' && steps.render.outputs.promoted_count != '0' && steps.pull_request.outputs.number != '' | |
| id: publication | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| PR_NUMBER: ${{ steps.pull_request.outputs.number }} | |
| PUBLICATION_BRANCH: ${{ steps.pull_request.outputs.branch }} | |
| run: | | |
| gh pr merge "$PR_NUMBER" --squash --delete-branch || gh pr merge "$PR_NUMBER" --squash --admin --delete-branch | |
| git fetch origin main:refs/remotes/origin/main | |
| pr_data=$(gh pr view "$PR_NUMBER" --json mergeCommit,mergedAt,url) | |
| published_sha=$(jq -r '.mergeCommit.oid // empty' <<< "$pr_data") | |
| pr_url=$(jq -r '.url' <<< "$pr_data") | |
| published_at=$(jq -r '.mergedAt // empty' <<< "$pr_data") | |
| test -n "$published_sha" | |
| test -n "$published_at" | |
| run_url="https://github.com/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" | |
| python3 scripts/sync_topic_plugins.py finalize-report \ | |
| --report-dir "${{ steps.render.outputs.report_dir }}" \ | |
| --commit-sha "$published_sha" \ | |
| --published-at "$published_at" \ | |
| --run-url "$run_url" \ | |
| --pr-url "$pr_url" \ | |
| --repository "$GITHUB_REPOSITORY" | |
| echo "published_sha=$published_sha" >> "$GITHUB_OUTPUT" | |
| echo "published_short_sha=${published_sha:0:12}" >> "$GITHUB_OUTPUT" | |
| git push origin --delete "$PUBLICATION_BRANCH" 2>/dev/null || true | |
| - name: Save the candidate ledger outside the public catalog | |
| if: steps.changes.outputs.changed == 'false' || steps.publication.outcome == 'success' | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: topic-candidate-ledger | |
| path: data/topic-candidates.json | |
| if-no-files-found: error | |
| retention-days: 30 | |
| - name: Save the successful collection report outside the public catalog | |
| if: steps.render.outputs.report_created == 'true' && steps.publication.outcome == 'success' | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: plugin-collection-report-${{ steps.publication.outputs.published_short_sha }}-${{ github.run_id }} | |
| path: ${{ steps.render.outputs.report_dir }} | |
| if-no-files-found: error | |
| retention-days: 90 | |
| - name: Reconcile the originating submission Issue | |
| if: always() && !cancelled() && github.event_name == 'workflow_dispatch' && inputs.mode == 'submission' && inputs.source_issue != '' && steps.assess.outcome == 'success' && steps.render.outcome == 'success' && steps.validate.outcome == 'success' && (steps.changes.outputs.changed == 'false' || steps.publication.outcome == 'success') | |
| env: | |
| GITHUB_TOKEN: ${{ github.token }} | |
| PROMOTED_COUNT: ${{ steps.render.outputs.promoted_count }} | |
| SOURCE_ISSUE: ${{ inputs.source_issue }} | |
| run: | | |
| if [ "$PROMOTED_COUNT" != "0" ]; then | |
| git fetch origin main | |
| git switch --detach origin/main | |
| python3 scripts/reconcile_submission_prs.py \ | |
| --issues --apply-published --only-issue "$SOURCE_ISSUE" \ | |
| --output source-issue-result.json | |
| else | |
| python3 scripts/reconcile_submission_prs.py \ | |
| --issues --apply-published --handle-exceptions --finalize-review \ | |
| --only-issue "$SOURCE_ISSUE" --output source-issue-result.json | |
| fi |