Skip to content

Commit 3e5693f

Browse files
DjDeveloperrDjDeveloperr
committed
Harden comment sessions against daemon restarts
1 parent 9535b7e commit 3e5693f

3 files changed

Lines changed: 508 additions & 39 deletions

File tree

‎actions/run-android-comment-session/action.yml‎

Lines changed: 178 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -324,6 +324,36 @@ runs:
324324
exit 1
325325
fi
326326
327+
cat > simdeck-daemon-supervisor.sh <<'EOF'
328+
#!/usr/bin/env bash
329+
set +e
330+
terminating=0
331+
child=""
332+
trap 'terminating=1; if [[ -n "${child}" ]]; then kill "${child}" 2>/dev/null; wait "${child}" 2>/dev/null; fi; exit 0' TERM INT HUP
333+
334+
while true; do
335+
"$@" >> simdeck-daemon.log 2>&1 &
336+
child="$!"
337+
echo "${child}" > simdeck-child.pid
338+
wait "${child}"
339+
status="$?"
340+
child=""
341+
342+
if [[ "${terminating}" -eq 1 ]]; then
343+
exit 0
344+
fi
345+
if [[ "${status}" -eq 75 || "${status}" -ge 128 ]]; then
346+
echo "[simdeck-action-supervisor] daemon exited with status ${status}; restarting" >> simdeck-daemon.log
347+
sleep 1
348+
continue
349+
fi
350+
351+
echo "[simdeck-action-supervisor] daemon exited with status ${status}; not restarting" >> simdeck-daemon.log
352+
exit "${status}"
353+
done
354+
EOF
355+
chmod +x simdeck-daemon-supervisor.sh
356+
327357
SIMDECK_VIDEO_CODEC=software \
328358
SIMDECK_ANDROID_VIDEO_CODEC=software \
329359
SIMDECK_ALLOWED_ORIGINS='*' \
@@ -334,6 +364,7 @@ runs:
334364
SIMDECK_REALTIME_MIN_BITRATE="${stream_min_bitrate}" \
335365
SIMDECK_REALTIME_BITS_PER_PIXEL="${stream_bits_per_pixel}" \
336366
SIMDECK_LOCAL_STREAM_FPS="${stream_fps}" \
367+
./simdeck-daemon-supervisor.sh \
337368
simdeck daemon run \
338369
--project-root "${GITHUB_WORKSPACE}" \
339370
--metadata-path "${metadata_path}" \
@@ -343,8 +374,7 @@ runs:
343374
--stream-quality "${SIMDECK_STREAM_PROFILE}" \
344375
--local-stream-fps "${stream_fps}" \
345376
--access-token "${access_token}" \
346-
--pairing-code 000000 \
347-
> simdeck-daemon.log 2>&1 &
377+
--pairing-code 000000 &
348378
echo "$!" > simdeck.pid
349379
350380
cloudflared tunnel --url "http://127.0.0.1:${SIMDECK_PORT}" --protocol http2 --no-autoupdate > cloudflared.log 2>&1 &
@@ -444,40 +474,106 @@ runs:
444474
sha="${{ steps.pr.outputs.sha }}"
445475
if [[ -n "${ARTIFACT_NAME_INPUT}" ]]; then
446476
artifact_name="${ARTIFACT_NAME_INPUT}"
477+
artifact_candidates="${artifact_name}"
447478
else
448479
artifact_name="${ARTIFACT_PREFIX}-${sha}"
480+
artifact_candidates="${artifact_name}"
481+
if [[ "${ARTIFACT_PREFIX}" != "${artifact_name}" ]]; then
482+
artifact_candidates+=$'\n'"${ARTIFACT_PREFIX}"
483+
fi
449484
fi
485+
printf '%s\n' "${artifact_candidates}" > /tmp/simdeck-artifact-candidates.txt
486+
artifact_candidates_summary="$(python3 - <<'PY'
487+
with open("/tmp/simdeck-artifact-candidates.txt", "r", encoding="utf-8") as handle:
488+
names = [line.strip() for line in handle if line.strip()]
489+
print(", ".join(f"`{name}`" for name in names))
490+
PY
491+
)"
492+
echo "SIMDECK_ARTIFACT_NAME=${artifact_name}" >> "${GITHUB_ENV}"
493+
echo "SIMDECK_ARTIFACT_CANDIDATES=${artifact_candidates_summary}" >> "${GITHUB_ENV}"
450494
mkdir -p downloaded-app
451495
rm -f /tmp/simdeck-artifact-download.status app-download.log
452496
453497
(
454498
set +e
455499
{
456-
run_id=""
457-
for attempt in {1..30}; do
458-
run_id="$(gh api -X GET "repos/${REPO}/actions/artifacts?name=${artifact_name}&per_page=100" \
459-
--jq '.artifacts[] | select(.expired == false) | .workflow_run.id' \
460-
| head -n 1 || true)"
461-
462-
if [[ -z "${run_id}" && -n "${BUILD_WORKFLOW}" ]]; then
463-
run_id="$(gh api --paginate "repos/${REPO}/actions/workflows/${BUILD_WORKFLOW}/runs?per_page=100" \
464-
--jq ".workflow_runs[] | select(.head_sha == \"${sha}\" and .conclusion == \"success\") | .id" \
465-
| head -n 1 || true)"
500+
find_artifact_by_run() {
501+
local candidate_run_id="$1"
502+
gh api -X GET "repos/${REPO}/actions/runs/${candidate_run_id}/artifacts?per_page=100" |
503+
python3 -c '
504+
import json
505+
import sys
506+
507+
with open("/tmp/simdeck-artifact-candidates.txt", "r", encoding="utf-8") as handle:
508+
names = [line.strip() for line in handle if line.strip()]
509+
510+
data = json.load(sys.stdin)
511+
for name in names:
512+
for artifact in data.get("artifacts", []):
513+
if artifact.get("expired") is False and artifact.get("name") == name:
514+
print(name)
515+
raise SystemExit(0)
516+
raise SystemExit(1)
517+
' || true
518+
}
519+
520+
find_artifact_record() {
521+
local artifact_record artifact_name_for_run candidate candidate_run_id
522+
523+
while IFS= read -r candidate; do
524+
[[ -z "${candidate}" ]] && continue
525+
artifact_record="$(gh api -X GET "repos/${REPO}/actions/artifacts?name=${candidate}&per_page=100" |
526+
SIMDECK_ARTIFACT_SHA="${sha}" python3 -c '
527+
import json
528+
import os
529+
import sys
530+
531+
data = json.load(sys.stdin)
532+
sha = os.environ["SIMDECK_ARTIFACT_SHA"]
533+
for artifact in data.get("artifacts", []):
534+
run = artifact.get("workflow_run") or {}
535+
if artifact.get("expired") is False and run.get("head_sha") == sha:
536+
print("{}\t{}".format(run.get("id"), artifact.get("name")))
537+
break
538+
' || true)"
539+
if [[ -n "${artifact_record}" ]]; then
540+
printf '%s\n' "${artifact_record}"
541+
return 0
542+
fi
543+
done < /tmp/simdeck-artifact-candidates.txt
544+
545+
if [[ -n "${BUILD_WORKFLOW}" ]]; then
546+
while IFS= read -r candidate_run_id; do
547+
[[ -z "${candidate_run_id}" ]] && continue
548+
artifact_name_for_run="$(find_artifact_by_run "${candidate_run_id}")"
549+
if [[ -n "${artifact_name_for_run}" ]]; then
550+
printf '%s\t%s\n' "${candidate_run_id}" "${artifact_name_for_run}"
551+
return 0
552+
fi
553+
done < <(gh api --paginate "repos/${REPO}/actions/workflows/${BUILD_WORKFLOW}/runs?per_page=100" \
554+
--jq ".workflow_runs[] | select(.head_sha == \"${sha}\" and .conclusion == \"success\") | .id" || true)
466555
fi
467556
468-
if [[ -n "${run_id}" ]]; then
469-
echo "Using build workflow run ${run_id} for ${sha}"
470-
gh run download "${run_id}" --repo "${REPO}" --name "${artifact_name}" --dir downloaded-app
557+
return 1
558+
}
559+
560+
for attempt in {1..30}; do
561+
artifact_record="$(find_artifact_record || true)"
562+
563+
if [[ -n "${artifact_record}" ]]; then
564+
IFS=$'\t' read -r run_id download_artifact_name <<< "${artifact_record}"
565+
echo "Using build workflow run ${run_id} artifact '${download_artifact_name}' for ${sha}"
566+
gh run download "${run_id}" --repo "${REPO}" --name "${download_artifact_name}" --dir downloaded-app
471567
exit_code="$?"
472568
echo "${exit_code}" > /tmp/simdeck-artifact-download.status
473569
exit "${exit_code}"
474570
fi
475571
476-
echo "Waiting for artifact '${artifact_name}' for PR head ${sha} (${attempt}/30)"
572+
echo "Waiting for artifact (${artifact_candidates_summary}) for PR head ${sha} (${attempt}/30)"
477573
sleep 20
478574
done
479575
480-
echo "No successful '${artifact_name}' artifact was found for PR head ${sha}." >&2
576+
echo "No successful unexpired artifact (${artifact_candidates_summary}) was found for PR head ${sha}." >&2
481577
echo "1" > /tmp/simdeck-artifact-download.status
482578
exit 1
483579
} > app-download.log 2>&1
@@ -543,6 +639,37 @@ runs:
543639
cat app-download.log
544640
status="$(cat /tmp/simdeck-artifact-download.status 2>/dev/null || echo 1)"
545641
if [[ "${status}" -ne 0 ]]; then
642+
echo "SIMDECK_SESSION_START_FAILED=1" >> "${GITHUB_ENV}"
643+
commit_sha="${{ steps.pr.outputs.sha }}"
644+
artifact_candidates="${SIMDECK_ARTIFACT_CANDIDATES:-${SIMDECK_ARTIFACT_NAME:-${ARTIFACT_PREFIX}-${commit_sha}}}"
645+
mention=""
646+
if [[ -n "${COMMAND_COMMENT_AUTHOR:-}" ]]; then
647+
mention="@${COMMAND_COMMENT_AUTHOR} "
648+
fi
649+
650+
cat > comment.md <<'EOF'
651+
__MENTION__SimDeck Android session could not start for commit `__COMMIT_SHA__`.
652+
653+
No unexpired APK artifact was available for this PR head. Expected one of: __ARTIFACT_CANDIDATES__.
654+
655+
Re-run the build workflow or push a new commit, then comment `simdeck run android` again.
656+
EOF
657+
658+
body="$(cat comment.md)"
659+
body="${body/__MENTION__/${mention}}"
660+
body="${body/__COMMIT_SHA__/${commit_sha}}"
661+
body="${body/__ARTIFACT_CANDIDATES__/${artifact_candidates}}"
662+
for attempt in {1..5}; do
663+
if [[ -n "${SIMDECK_STATUS_COMMENT_ID:-}" ]]; then
664+
if gh api -X PATCH "repos/${REPO}/issues/comments/${SIMDECK_STATUS_COMMENT_ID}" -f body="${body}"; then
665+
break
666+
fi
667+
elif comment_id="$(gh api "repos/${REPO}/issues/${PR_NUMBER}/comments" -f body="${body}" --jq '.id')"; then
668+
echo "SIMDECK_STATUS_COMMENT_ID=${comment_id}" >> "${GITHUB_ENV}"
669+
break
670+
fi
671+
sleep $((attempt * 5))
672+
done
546673
exit "${status}"
547674
fi
548675
@@ -592,6 +719,7 @@ runs:
592719
if [[ -f /tmp/sim-boot-start && -f /tmp/sim-boot-end ]]; then
593720
echo "Android emulator boot took $(( $(cat /tmp/sim-boot-end) - $(cat /tmp/sim-boot-start) )) seconds."
594721
fi
722+
echo "SIMDECK_SESSION_OPEN=1" >> "${GITHUB_ENV}"
595723
596724
- name: Update status comment after app launch
597725
shell: bash
@@ -621,14 +749,39 @@ runs:
621749
set -euo pipefail
622750
udid="${{ steps.android.outputs.udid }}"
623751
end=$((SECONDS + KEEPALIVE_SECONDS))
752+
SIMDECK_DAEMON_HEALTH_GRACE_SECONDS="${SIMDECK_DAEMON_HEALTH_GRACE_SECONDS:-90}"
753+
health_failure_started=""
754+
755+
note_daemon_health_failure() {
756+
local reason="$1"
757+
if [[ -z "${health_failure_started}" ]]; then
758+
health_failure_started="${SECONDS}"
759+
fi
760+
761+
if (( SECONDS - health_failure_started >= SIMDECK_DAEMON_HEALTH_GRACE_SECONDS )); then
762+
echo "${reason} for ${SIMDECK_DAEMON_HEALTH_GRACE_SECONDS}s; stopping session." >&2
763+
cat simdeck-list-error.log >&2 || true
764+
cat simdeck-health.log >&2 || true
765+
cat simdeck-daemon.log >&2 || true
766+
return 1
767+
fi
768+
769+
echo "${reason}; waiting for SimDeck daemon supervisor to recover."
770+
sleep 5
771+
return 0
772+
}
624773
625774
while (( SECONDS < end )); do
626775
if [[ -f simdeck.pid ]] && ! kill -0 "$(cat simdeck.pid)" 2>/dev/null; then
627776
echo "SimDeck daemon process exited; stopping session."
777+
cat simdeck-daemon.log >&2 || true
628778
exit 1
629779
fi
630780
631-
list_json="$(simdeck --server-url "http://127.0.0.1:${SIMDECK_PORT}" list --format json)"
781+
if ! list_json="$(simdeck --server-url "http://127.0.0.1:${SIMDECK_PORT}" list --format json 2>simdeck-list-error.log)"; then
782+
note_daemon_health_failure "SimDeck daemon list check failed" || exit 1
783+
continue
784+
fi
632785
if ! SIMDECK_LIST_JSON="${list_json}" python3 - "${udid}" <<'PY'
633786
import json
634787
import os
@@ -647,7 +800,12 @@ runs:
647800
exit 0
648801
fi
649802
650-
curl -fsS "http://127.0.0.1:${SIMDECK_PORT}/api/health?simdeckToken=${{ steps.stream.outputs.access_token }}" >/dev/null
803+
if ! curl -fsS "http://127.0.0.1:${SIMDECK_PORT}/api/health?simdeckToken=${{ steps.stream.outputs.access_token }}" >/dev/null 2>simdeck-health.log; then
804+
note_daemon_health_failure "SimDeck daemon health check failed" || exit 1
805+
continue
806+
fi
807+
808+
health_failure_started=""
651809
sleep 15
652810
done
653811
@@ -668,7 +826,7 @@ runs:
668826
simdeck daemon stop
669827
670828
- name: Update status comment at end
671-
if: always()
829+
if: always() && env.SIMDECK_SESSION_OPEN == '1'
672830
shell: bash
673831
run: |
674832
set -euo pipefail

0 commit comments

Comments
 (0)