Preserve harvest diagnostics in PR run

Author: wang177777

.github/workflows/jama_network_open_harvest.yml

@@ -36,41 +36,49 @@ jobs:
         run: python -m pip install --disable-pip-version-check requests beautifulsoup4 lxml
 
       - name: Harvest and reconcile records
-        run: python scripts/fetch_jama_network_open.py
+        id: harvest
+        shell: bash
+        run: |
+          mkdir -p "$OUT_DIR"
+          set +e
+          python scripts/fetch_jama_network_open.py > "$OUT_DIR/run.log" 2>&1
+          code=$?
+          echo "exit_code=$code" >> "$GITHUB_OUTPUT"
+          cat "$OUT_DIR/run.log"
+          exit 0
 
       - name: Write run metadata
+        if: always()
         shell: bash
         run: |
           python - <<'PY'
           import json, os
           from pathlib import Path
           out = Path(os.environ['OUT_DIR'])
-          summary = json.loads((out / 'summary.json').read_text(encoding='utf-8'))
+          summary_path = out / 'summary.json'
+          summary = json.loads(summary_path.read_text(encoding='utf-8')) if summary_path.exists() else {}
           payload = {
               'run_id': int(os.environ['GITHUB_RUN_ID']),
               'run_attempt': int(os.environ['GITHUB_RUN_ATTEMPT']),
               'commit_sha': os.environ['GITHUB_SHA'],
               'artifact_name': 'jama-network-open-six-month-literature',
+              'harvest_exit_code': '${{ steps.harvest.outputs.exit_code }}',
               'summary': summary,
           }
           (out / 'latest_run.json').write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding='utf-8')
           PY
 
       - name: Upload complete data artifact
+        if: always()
         uses: actions/upload-artifact@v4
         with:
           name: jama-network-open-six-month-literature
-          path: |
-            jama_output/research_papers.csv
-            jama_output/all_records.csv
-            jama_output/summary.json
-            jama_output/preview_25.csv
-            jama_output/latest_run.json
+          path: jama_output/**
           if-no-files-found: error
           retention-days: 7
 
       - name: Publish run pointer and preview
-        if: github.event_name == 'push'
+        if: github.event_name == 'push' && steps.harvest.outputs.exit_code == '0'
         shell: bash
         run: |
           git config user.name 'github-actions[bot]'
@@ -80,3 +88,7 @@ jobs:
             git commit -m 'Update JAMA Network Open harvest summary'
             git push
           fi
+
+      - name: Fail workflow when harvest failed
+        if: steps.harvest.outputs.exit_code != '0'
+        run: exit 1