Run diagnostic JAMA Network Open literature harvest #11
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Harvest JAMA Network Open literature | |
| on: | |
| workflow_dispatch: | |
| push: | |
| paths: | |
| - '.github/workflows/jama_network_open_harvest.yml' | |
| - 'scripts/fetch_jama_network_open.py' | |
| - 'jama_trigger.txt' | |
| pull_request: | |
| paths: | |
| - 'scripts/fetch_jama_network_open.py' | |
| - 'jama_trigger_pr.txt' | |
| permissions: | |
| contents: write | |
| jobs: | |
| harvest: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 30 | |
| env: | |
| START_DATE: '2026-01-24' | |
| END_DATE: '2026-07-24' | |
| OUT_DIR: 'jama_output' | |
| steps: | |
| - name: Check out repository | |
| uses: actions/checkout@v4 | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.12' | |
| - name: Install dependencies | |
| run: python -m pip install --disable-pip-version-check requests beautifulsoup4 lxml | |
| - name: Patch Crossref query compatibility | |
| shell: bash | |
| run: | | |
| sed -i '/"cursor-max": rows,/d' scripts/fetch_jama_network_open.py | |
| sed -i 's/,type:journal-article//' scripts/fetch_jama_network_open.py | |
| - name: Harvest and reconcile records | |
| id: harvest | |
| shell: bash | |
| run: | | |
| mkdir -p "$OUT_DIR" | |
| set +e | |
| python scripts/fetch_jama_network_open.py > "$OUT_DIR/run.log" 2>&1 | |
| code=$? | |
| echo "exit_code=$code" >> "$GITHUB_OUTPUT" | |
| cat "$OUT_DIR/run.log" | |
| exit 0 | |
| - name: Write run metadata | |
| if: always() | |
| shell: bash | |
| run: | | |
| python - <<'PY' | |
| import json, os | |
| from pathlib import Path | |
| out = Path(os.environ['OUT_DIR']) | |
| summary_path = out / 'summary.json' | |
| summary = json.loads(summary_path.read_text(encoding='utf-8')) if summary_path.exists() else {} | |
| payload = { | |
| 'run_id': int(os.environ['GITHUB_RUN_ID']), | |
| 'run_attempt': int(os.environ['GITHUB_RUN_ATTEMPT']), | |
| 'commit_sha': os.environ['GITHUB_SHA'], | |
| 'artifact_name': 'jama-network-open-six-month-literature', | |
| 'harvest_exit_code': '${{ steps.harvest.outputs.exit_code }}', | |
| 'summary': summary, | |
| } | |
| (out / 'latest_run.json').write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding='utf-8') | |
| PY | |
| - name: Upload complete data artifact | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: jama-network-open-six-month-literature | |
| path: jama_output/** | |
| if-no-files-found: error | |
| retention-days: 7 | |
| - name: Publish run pointer and preview | |
| if: github.event_name == 'push' && steps.harvest.outputs.exit_code == '0' | |
| shell: bash | |
| run: | | |
| git config user.name 'github-actions[bot]' | |
| git config user.email '41898282+github-actions[bot]@users.noreply.github.com' | |
| git add jama_output/latest_run.json jama_output/summary.json jama_output/preview_25.csv | |
| if ! git diff --cached --quiet; then | |
| git commit -m 'Update JAMA Network Open harvest summary' | |
| git push | |
| fi | |
| - name: Fail workflow when harvest failed | |
| if: steps.harvest.outputs.exit_code != '0' | |
| run: exit 1 |