Skip to content

Commit cd010b7

Browse files
PigbibiPigbibi
committed
Add shared monitor dispatcher scheduler
1 parent 363680d commit cd010b7

7 files changed

Lines changed: 549 additions & 39 deletions

File tree

‎.github/workflows/sync-cloud-run-env.yml‎

Lines changed: 147 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ env:
1111
GCP_WORKLOAD_IDENTITY_PROVIDER: projects/1088907247379/locations/global/workloadIdentityPools/github-actions/providers/github-main
1212
GCP_WORKLOAD_IDENTITY_SERVICE_ACCOUNT: firstrade-platform-deploy@firstradequant.iam.gserviceaccount.com
1313
GCP_RUNTIME_SERVICE_ACCOUNT: firstrade-platform-runtime@firstradequant.iam.gserviceaccount.com
14+
GCP_SCHEDULER_SERVICE_ACCOUNT: firstrade-platform-scheduler@firstradequant.iam.gserviceaccount.com
1415
GCP_ARTIFACT_REGISTRY_HOSTNAME: us-central1-docker.pkg.dev
1516
GCP_ARTIFACT_REGISTRY_REPOSITORY: cloud-run-source-deploy
1617

@@ -587,6 +588,33 @@ jobs:
587588
add_optional_secret FIRSTRADE_MFA_PHONE FIRSTRADE_MFA_PHONE_SECRET_NAME FIRSTRADE_MFA_PHONE
588589
add_optional_secret FIRSTRADE_MFA_CODE FIRSTRADE_MFA_CODE_SECRET_NAME FIRSTRADE_MFA_CODE
589590
591+
service_url="$(gcloud run services describe "${CLOUD_RUN_SERVICE}" \
592+
--project="${GCP_PROJECT_ID}" \
593+
--region="${CLOUD_RUN_REGION}" \
594+
--format='value(status.url)' 2>/dev/null || true)"
595+
if [ -z "${service_url}" ]; then
596+
echo "Unable to resolve Cloud Run service URL for ${CLOUD_RUN_SERVICE}; cannot sync monitor dispatcher targets." >&2
597+
exit 1
598+
fi
599+
monitor_targets_json="$(
600+
SERVICE_URL="${service_url}" python - <<'PY'
601+
import json
602+
import os
603+
604+
runtime_target = json.loads(os.environ.get("RUNTIME_TARGET_JSON") or "{}")
605+
target = {
606+
"service_name": os.environ.get("CLOUD_RUN_SERVICE"),
607+
"service_url": os.environ.get("SERVICE_URL"),
608+
"strategy_profile": runtime_target.get("strategy_profile"),
609+
"account_scope": runtime_target.get("account_scope"),
610+
"runtime_target_enabled": os.environ.get("RUNTIME_TARGET_ENABLED", "true"),
611+
"scheduler": runtime_target.get("scheduler") if isinstance(runtime_target.get("scheduler"), dict) else {},
612+
}
613+
print(json.dumps({"targets": [target]}, separators=(",", ":")))
614+
PY
615+
)"
616+
env_pairs+=("MONITOR_DISPATCH_TARGETS_JSON=${monitor_targets_json}")
617+
590618
gcloud_args=(
591619
run services update "${CLOUD_RUN_SERVICE}"
592620
--region "${CLOUD_RUN_REGION}"
@@ -655,46 +683,42 @@ jobs:
655683
exit 1
656684
fi
657685
658-
for suffix in scheduler probe-scheduler precheck-scheduler; do
659-
case "${suffix}" in
660-
scheduler)
661-
schedule_time="${main_time}"
662-
scheduler_path="/run"
663-
;;
664-
probe-scheduler)
665-
schedule_time="${probe_time}"
666-
scheduler_path="/probe"
667-
;;
668-
precheck-scheduler)
669-
schedule_time="${precheck_time}"
670-
scheduler_path="/dry-run"
671-
;;
672-
esac
673-
674-
scheduler_job_candidates=("${CLOUD_RUN_SERVICE}-${suffix}")
675-
if [[ "${CLOUD_RUN_SERVICE}" == *-service ]]; then
676-
scheduler_job_candidates+=("${CLOUD_RUN_SERVICE%-service}-${suffix}")
677-
fi
686+
gcloud run services add-iam-policy-binding "${CLOUD_RUN_SERVICE}" \
687+
--project="${GCP_PROJECT_ID}" \
688+
--region="${CLOUD_RUN_REGION}" \
689+
--member="serviceAccount:${GCP_SCHEDULER_SERVICE_ACCOUNT}" \
690+
--role="roles/run.invoker" \
691+
--quiet
692+
gcloud run services add-iam-policy-binding "${CLOUD_RUN_SERVICE}" \
693+
--project="${GCP_PROJECT_ID}" \
694+
--region="${CLOUD_RUN_REGION}" \
695+
--member="serviceAccount:${GCP_RUNTIME_SERVICE_ACCOUNT}" \
696+
--role="roles/run.invoker" \
697+
--quiet
678698
679-
job_name=""
680-
current_schedule=""
681-
for candidate_job in "${scheduler_job_candidates[@]}"; do
682-
current_schedule="$(gcloud scheduler jobs describe "${candidate_job}" \
683-
--project="${GCP_PROJECT_ID}" \
684-
--location="${scheduler_location}" \
685-
--format='value(schedule)' 2>/dev/null || true)"
686-
if [ -n "${current_schedule}" ]; then
687-
job_name="${candidate_job}"
688-
break
689-
fi
690-
done
699+
scheduler_job_candidates=("${CLOUD_RUN_SERVICE}-scheduler")
700+
if [[ "${CLOUD_RUN_SERVICE}" == *-service ]]; then
701+
scheduler_job_candidates+=("${CLOUD_RUN_SERVICE%-service}-scheduler")
702+
fi
691703
692-
if [ -z "${current_schedule}" ]; then
693-
echo "Cloud Scheduler job for ${CLOUD_RUN_SERVICE} ${suffix} was not found in ${scheduler_location}; skipping schedule sync."
694-
continue
704+
job_name=""
705+
current_schedule=""
706+
for candidate_job in "${scheduler_job_candidates[@]}"; do
707+
current_schedule="$(gcloud scheduler jobs describe "${candidate_job}" \
708+
--project="${GCP_PROJECT_ID}" \
709+
--location="${scheduler_location}" \
710+
--format='value(schedule)' 2>/dev/null || true)"
711+
if [ -n "${current_schedule}" ]; then
712+
job_name="${candidate_job}"
713+
break
695714
fi
715+
done
716+
if [ -z "${job_name}" ]; then
717+
job_name="${scheduler_job_candidates[0]}"
718+
fi
696719
697-
desired_schedule="$(CURRENT_SCHEDULE="${current_schedule}" SCHEDULE_TIME="${schedule_time}" python - <<'PY'
720+
if [ -n "${current_schedule}" ]; then
721+
desired_schedule="$(CURRENT_SCHEDULE="${current_schedule}" SCHEDULE_TIME="${main_time}" python - <<'PY'
698722
import os
699723
700724
current_fields = os.environ["CURRENT_SCHEDULE"].split()
@@ -711,8 +735,25 @@ jobs:
711735
)
712736
PY
713737
)"
738+
else
739+
desired_schedule="$(SCHEDULE_TIME="${main_time}" python - <<'PY'
740+
import os
714741
715-
scheduler_uri="${service_url}${scheduler_path}"
742+
fields = os.environ["SCHEDULE_TIME"].split()
743+
if len(fields) == 5:
744+
print(" ".join(fields))
745+
elif len(fields) == 2:
746+
print(" ".join([*fields, "*", "*", "*"]))
747+
else:
748+
raise SystemExit(
749+
f"Cloud Scheduler override must have 2 time fields or 5 cron fields: {os.environ['SCHEDULE_TIME']!r}"
750+
)
751+
PY
752+
)"
753+
fi
754+
755+
scheduler_uri="${service_url}/run"
756+
if [ -n "${current_schedule}" ]; then
716757
echo "Updating Cloud Scheduler job ${job_name} schedule to ${desired_schedule}, timezone to ${market_timezone}, and URI to ${scheduler_uri}."
717758
gcloud scheduler jobs update http "${job_name}" \
718759
--project="${GCP_PROJECT_ID}" \
@@ -721,6 +762,75 @@ jobs:
721762
--schedule="${desired_schedule}" \
722763
--time-zone="${market_timezone}" \
723764
--quiet
765+
else
766+
echo "Creating Cloud Scheduler job ${job_name} schedule ${desired_schedule}, timezone ${market_timezone}, and URI ${scheduler_uri}."
767+
gcloud scheduler jobs create http "${job_name}" \
768+
--project="${GCP_PROJECT_ID}" \
769+
--location="${scheduler_location}" \
770+
--uri="${scheduler_uri}" \
771+
--http-method=POST \
772+
--oidc-service-account-email="${GCP_SCHEDULER_SERVICE_ACCOUNT}" \
773+
--oidc-token-audience="${service_url}" \
774+
--schedule="${desired_schedule}" \
775+
--time-zone="${market_timezone}" \
776+
--attempt-deadline=600s \
777+
--quiet
778+
fi
779+
780+
monitor_job_name="firstrade-monitor-dispatcher-scheduler"
781+
monitor_uri="${service_url}/monitor-dispatch"
782+
if gcloud scheduler jobs describe "${monitor_job_name}" \
783+
--project="${GCP_PROJECT_ID}" \
784+
--location="${scheduler_location}" >/dev/null 2>&1; then
785+
echo "Updating Cloud Scheduler job ${monitor_job_name} to ${monitor_uri}."
786+
gcloud scheduler jobs update http "${monitor_job_name}" \
787+
--project="${GCP_PROJECT_ID}" \
788+
--location="${scheduler_location}" \
789+
--uri="${monitor_uri}" \
790+
--http-method=POST \
791+
--oidc-service-account-email="${GCP_SCHEDULER_SERVICE_ACCOUNT}" \
792+
--oidc-token-audience="${service_url}" \
793+
--schedule="*/5 * * * *" \
794+
--time-zone="UTC" \
795+
--attempt-deadline=180s \
796+
--quiet
797+
else
798+
echo "Creating Cloud Scheduler job ${monitor_job_name} at ${monitor_uri}."
799+
gcloud scheduler jobs create http "${monitor_job_name}" \
800+
--project="${GCP_PROJECT_ID}" \
801+
--location="${scheduler_location}" \
802+
--uri="${monitor_uri}" \
803+
--http-method=POST \
804+
--oidc-service-account-email="${GCP_SCHEDULER_SERVICE_ACCOUNT}" \
805+
--oidc-token-audience="${service_url}" \
806+
--schedule="*/5 * * * *" \
807+
--time-zone="UTC" \
808+
--attempt-deadline=180s \
809+
--quiet
810+
fi
811+
812+
legacy_jobs=(
813+
"${CLOUD_RUN_SERVICE}-probe-scheduler"
814+
"${CLOUD_RUN_SERVICE}-precheck-scheduler"
815+
"${CLOUD_RUN_SERVICE}-session-check-scheduler"
816+
)
817+
if [[ "${CLOUD_RUN_SERVICE}" == *-service ]]; then
818+
legacy_jobs+=(
819+
"${CLOUD_RUN_SERVICE%-service}-probe-scheduler"
820+
"${CLOUD_RUN_SERVICE%-service}-precheck-scheduler"
821+
"${CLOUD_RUN_SERVICE%-service}-session-check-scheduler"
822+
)
823+
fi
824+
for legacy_job in "${legacy_jobs[@]}"; do
825+
if gcloud scheduler jobs describe "${legacy_job}" \
826+
--project="${GCP_PROJECT_ID}" \
827+
--location="${scheduler_location}" >/dev/null 2>&1; then
828+
echo "Deleting legacy Cloud Scheduler job ${legacy_job}; monitor dispatcher now owns probe/precheck."
829+
gcloud scheduler jobs delete "${legacy_job}" \
830+
--project="${GCP_PROJECT_ID}" \
831+
--location="${scheduler_location}" \
832+
--quiet
833+
fi
724834
done
725835
726836
- name: Clean up old Cloud Run revisions and images

0 commit comments

Comments
 (0)