Skip to content

Publish Snapshot Artifacts #77

Publish Snapshot Artifacts

Publish Snapshot Artifacts #77

name: Publish Snapshot Artifacts
on:
schedule:
# Weekday Global ETF snapshot refresh after the regular US close.
- cron: '30 22 * * 1-5'
workflow_run:
workflows: ["Update Source Input Data"]
types: [completed]
workflow_dispatch:
inputs:
profile:
description: Snapshot profile to build
required: true
default: russell_top50_leader_rotation
type: choice
options:
- russell_top50_leader_rotation
- global_etf_rotation
use_sample_data:
description: Generate synthetic input data for a workflow smoke test
required: true
default: false
type: boolean
prices_path:
description: Local, gs://, or http(s) price history path. Required unless use_sample_data=true.
required: false
type: string
universe_path:
description: Local, gs://, or http(s) universe path. Required unless use_sample_data=true.
required: false
type: string
source_input_manifest_path:
description: Optional local, gs://, or http(s) source input manifest JSON.
required: false
type: string
as_of_date:
description: Optional snapshot date, for example 2026-04-01
required: false
type: string
artifact_dir:
description: Optional artifact output directory; defaults to data/output/<profile>
required: false
type: string
gcs_prefix:
description: Optional GCS prefix. Leave blank to only build and upload GitHub artifact.
required: false
type: string
execute_publish:
description: Actually upload to GCS. false only prints the publish plan.
required: true
default: false
type: boolean
config_path:
description: Optional local, gs://, or http(s) strategy config path. Leave blank for bundled/default lookup.
required: false
type: string
current_holdings:
description: Optional comma-separated holdings used only for hold-bonus preview.
required: false
type: string
portfolio_total_equity:
description: Optional equity for dynamic position-count preview.
required: false
type: string
min_adv20_usd:
description: Optional Russell min ADV override.
required: false
type: string
concurrency:
group: ${{ github.workflow }}-${{ github.ref_name }}
cancel-in-progress: false
jobs:
build-and-publish:
if: github.event_name != 'workflow_run' || (github.event.workflow_run.conclusion == 'success' && github.event.workflow_run.event == 'schedule')
runs-on: ubuntu-latest
timeout-minutes: 60
strategy:
fail-fast: false
matrix:
profile: ${{ fromJSON(github.event_name == 'schedule' && '["global_etf_rotation"]' || (github.event_name != 'workflow_dispatch' && '["russell_top50_leader_rotation"]' || format('["{0}"]', inputs.profile || 'russell_top50_leader_rotation'))) }}
permissions:
actions: read
contents: read
id-token: write
env:
GCP_PROJECT_ID: ${{ vars.GCP_PROJECT_ID || 'interactivebrokersquant' }}
GCP_WORKLOAD_IDENTITY_PROVIDER: ${{ vars.GCP_WORKLOAD_IDENTITY_PROVIDER }}
GCP_WORKLOAD_IDENTITY_SERVICE_ACCOUNT: ${{ vars.GCP_WORKLOAD_IDENTITY_SERVICE_ACCOUNT }}
PROFILE: ${{ matrix.profile }}
USE_SAMPLE_DATA: ${{ inputs.use_sample_data }}
PRICES_PATH: ${{ inputs.prices_path }}
UNIVERSE_PATH: ${{ inputs.universe_path }}
SOURCE_INPUT_MANIFEST_PATH: ${{ inputs.source_input_manifest_path }}
AS_OF_DATE: ${{ inputs.as_of_date }}
ARTIFACT_DIR_INPUT: ${{ inputs.artifact_dir }}
GCS_PREFIX: ${{ inputs.gcs_prefix }}
EXECUTE_PUBLISH: ${{ inputs.execute_publish }}
CONFIG_PATH: ${{ inputs.config_path }}
CURRENT_HOLDINGS: ${{ inputs.current_holdings }}
PORTFOLIO_TOTAL_EQUITY: ${{ inputs.portfolio_total_equity }}
MIN_ADV20_USD: ${{ inputs.min_adv20_usd }}
SCHEDULED_PRICES_PATH: ${{ vars.SCHEDULED_US_EQUITY_PRICES_PATH || 'gs://qsl-runtime-logs-shared/strategy-artifacts/us_equity/inputs/r1000_official_monthly_v2_alias/r1000_price_history.csv' }}
SCHEDULED_UNIVERSE_PATH: ${{ vars.SCHEDULED_US_EQUITY_UNIVERSE_PATH || 'gs://qsl-runtime-logs-shared/strategy-artifacts/us_equity/inputs/r1000_official_monthly_v2_alias/r1000_universe_history.csv' }}
SCHEDULED_SOURCE_INPUT_MANIFEST_PATH: ${{ vars.SCHEDULED_US_EQUITY_SOURCE_INPUT_MANIFEST_PATH || 'gs://qsl-runtime-logs-shared/strategy-artifacts/us_equity/inputs/r1000_official_monthly_v2_alias/r1000_source_input_manifest.json' }}
SCHEDULED_RUSSELL_TOP50_LEADER_ROTATION_PRICES_PATH: ${{ vars.SCHEDULED_RUSSELL_TOP50_LEADER_ROTATION_PRICES_PATH || vars.SCHEDULED_US_EQUITY_PRICES_PATH || 'gs://qsl-runtime-logs-shared/strategy-artifacts/us_equity/inputs/r1000_official_monthly_v2_alias/r1000_price_history.csv' }}
SCHEDULED_RUSSELL_TOP50_LEADER_ROTATION_UNIVERSE_PATH: ${{ vars.SCHEDULED_RUSSELL_TOP50_LEADER_ROTATION_UNIVERSE_PATH || 'gs://qsl-runtime-logs-shared/strategy-artifacts/us_equity/inputs/r1000_official_monthly_v2_alias/r1000_latest_holdings_snapshot.csv' }}
SCHEDULED_RUSSELL_TOP50_LEADER_ROTATION_GCS_PREFIX: ${{ vars.SCHEDULED_RUSSELL_TOP50_LEADER_ROTATION_GCS_PREFIX || 'gs://qsl-runtime-logs-shared/strategy-artifacts/us_equity/russell_top50_leader_rotation_staging' }}
SCHEDULED_GLOBAL_ETF_ROTATION_GCS_PREFIX: ${{ vars.SCHEDULED_GLOBAL_ETF_ROTATION_GCS_PREFIX || 'gs://qsl-runtime-logs-shared/strategy-artifacts/us_equity/global_etf_rotation' }}
steps:
- name: Checkout
uses: actions/checkout@v6
- name: Verify main CI succeeded before publish
if: >-
github.event_name == 'schedule' ||
github.event_name == 'workflow_run' ||
(github.event_name == 'workflow_dispatch' && inputs.execute_publish == true)
env:
GH_TOKEN: ${{ github.token }}
run: bash .github/scripts/verify_main_ci_success.sh
- name: Set up Python
uses: actions/setup-python@v6
with:
python-version: '3.11'
- name: Install package
run: |
set -euo pipefail
python -m pip install --upgrade pip
python -m pip install -e .
- name: Apply scheduled defaults
run: |
set -euo pipefail
profile="${PROFILE:-russell_top50_leader_rotation}"
use_sample_data="${USE_SAMPLE_DATA:-false}"
prices_path="${PRICES_PATH:-}"
universe_path="${UNIVERSE_PATH:-}"
source_input_manifest_path="${SOURCE_INPUT_MANIFEST_PATH:-}"
config_path="${CONFIG_PATH:-}"
gcs_prefix="${GCS_PREFIX:-}"
execute_publish="${EXECUTE_PUBLISH:-false}"
if [ "${GITHUB_EVENT_NAME}" = "schedule" ] || [ "${GITHUB_EVENT_NAME}" = "workflow_run" ]; then
profile="${PROFILE}"
use_sample_data="false"
prices_path="${PRICES_PATH:-${SCHEDULED_PRICES_PATH}}"
universe_path="${UNIVERSE_PATH:-${SCHEDULED_UNIVERSE_PATH}}"
source_input_manifest_path="${SOURCE_INPUT_MANIFEST_PATH:-${SCHEDULED_SOURCE_INPUT_MANIFEST_PATH}}"
case "${profile}" in
russell_top50_leader_rotation)
prices_path="${PRICES_PATH:-${SCHEDULED_RUSSELL_TOP50_LEADER_ROTATION_PRICES_PATH}}"
universe_path="${UNIVERSE_PATH:-${SCHEDULED_RUSSELL_TOP50_LEADER_ROTATION_UNIVERSE_PATH}}"
config_path=""
gcs_prefix="${GCS_PREFIX:-${SCHEDULED_RUSSELL_TOP50_LEADER_ROTATION_GCS_PREFIX}}"
;;
global_etf_rotation)
prices_path="${PRICES_PATH:-}"
universe_path="${UNIVERSE_PATH:-}"
source_input_manifest_path="${SOURCE_INPUT_MANIFEST_PATH:-}"
config_path=""
gcs_prefix="${GCS_PREFIX:-${SCHEDULED_GLOBAL_ETF_ROTATION_GCS_PREFIX}}"
;;
*)
echo "Unsupported scheduled profile: ${profile}" >&2
exit 1
;;
esac
execute_publish="${EXECUTE_PUBLISH:-true}"
fi
{
echo "PROFILE=${profile}"
echo "USE_SAMPLE_DATA=${use_sample_data}"
echo "PRICES_PATH=${prices_path}"
echo "UNIVERSE_PATH=${universe_path}"
echo "SOURCE_INPUT_MANIFEST_PATH=${source_input_manifest_path}"
echo "CONFIG_PATH=${config_path}"
echo "GCS_PREFIX=${gcs_prefix}"
echo "EXECUTE_PUBLISH=${execute_publish}"
} >> "$GITHUB_ENV"
{
echo "profile=${profile}"
echo "use_sample_data=${use_sample_data}"
echo "prices_path=${prices_path:-<empty>}"
echo "universe_path=${universe_path:-<empty>}"
echo "source_input_manifest_path=${source_input_manifest_path:-<empty>}"
echo "config_path=${config_path:-<empty>}"
echo "gcs_prefix=${gcs_prefix:-<empty>}"
echo "execute_publish=${execute_publish}"
}
- name: Check whether Google Cloud auth is needed
id: gcp-auth-check
run: |
set -euo pipefail
needed=false
for value in "${PRICES_PATH:-}" "${UNIVERSE_PATH:-}" "${CONFIG_PATH:-}" "${SOURCE_INPUT_MANIFEST_PATH:-}"; do
case "${value}" in
gs://*) needed=true ;;
esac
done
if [ "${EXECUTE_PUBLISH}" = "true" ] && [ -n "${GCS_PREFIX:-}" ]; then
needed=true
fi
echo "needed=${needed}" >> "$GITHUB_OUTPUT"
- name: Authenticate to Google Cloud
if: steps.gcp-auth-check.outputs.needed == 'true'
uses: google-github-actions/auth@v3
with:
workload_identity_provider: ${{ env.GCP_WORKLOAD_IDENTITY_PROVIDER }}
service_account: ${{ env.GCP_WORKLOAD_IDENTITY_SERVICE_ACCOUNT }}
- name: Set up gcloud
if: steps.gcp-auth-check.outputs.needed == 'true'
uses: google-github-actions/setup-gcloud@v3
with:
project_id: ${{ env.GCP_PROJECT_ID }}
version: '>= 416.0.0'
- name: Build snapshot artifacts
id: build
run: |
set -euo pipefail
if [ "${USE_SAMPLE_DATA}" = "true" ]; then
sample_input_dir="data/input/sample/${PROFILE}"
python scripts/write_sample_input_data.py --profile "${PROFILE}" --output-dir "${sample_input_dir}"
PRICES_PATH="${sample_input_dir}/prices.csv"
UNIVERSE_PATH="${sample_input_dir}/universe.csv"
elif [ "${PROFILE}" = "global_etf_rotation" ] && [ -z "${PRICES_PATH:-}" ]; then
echo "No Global ETF prices_path supplied; builder will download current Yahoo chart history."
else
resolved_input_dir="data/input/resolved/${PROFILE}"
resolved_env="${RUNNER_TEMP:-/tmp}/resolved_snapshot_inputs.env"
resolve_args=(
--prices "${PRICES_PATH}"
--universe "${UNIVERSE_PATH}"
--output-dir "${resolved_input_dir}"
--env-output "${resolved_env}"
)
if [ -n "${CONFIG_PATH:-}" ]; then
resolve_args+=(--config "${CONFIG_PATH}")
fi
if [ -n "${SOURCE_INPUT_MANIFEST_PATH:-}" ]; then
resolve_args+=(--source-input-manifest "${SOURCE_INPUT_MANIFEST_PATH}")
fi
python scripts/resolve_snapshot_inputs.py "${resolve_args[@]}"
# shellcheck disable=SC1090
source "${resolved_env}"
fi
if [ -z "${PRICES_PATH:-}" ] || [ ! -f "${PRICES_PATH}" ]; then
if [ "${PROFILE}" != "global_etf_rotation" ]; then
echo "prices_path is required and must resolve to a local file unless use_sample_data=true: ${PRICES_PATH:-<empty>}" >&2
exit 1
fi
fi
if [ -z "${UNIVERSE_PATH:-}" ] || [ ! -f "${UNIVERSE_PATH}" ]; then
if [ "${PROFILE}" != "global_etf_rotation" ]; then
echo "universe_path is required and must resolve to a local file unless use_sample_data=true: ${UNIVERSE_PATH:-<empty>}" >&2
exit 1
fi
fi
artifact_dir="${ARTIFACT_DIR_INPUT:-data/output/${PROFILE}}"
mkdir -p "${artifact_dir}"
{
echo "artifact_dir=${artifact_dir}"
echo "artifact_name=us-equity-snapshot-${PROFILE}-${GITHUB_RUN_ID}"
} >> "$GITHUB_OUTPUT"
should_publish="true"
publish_reason="not_a_scheduled_monthly_profile"
snapshot_as_of=""
month_end_trading_day=""
calendar_source=""
if [ "${GITHUB_EVENT_NAME}" = "schedule" ] || [ "${GITHUB_EVENT_NAME}" = "workflow_run" ]; then
case "${PROFILE}" in
russell_top50_leader_rotation)
monthly_publish_env="${RUNNER_TEMP:-/tmp}/monthly_snapshot_publish_window.env"
monthly_args=(
--prices "${PRICES_PATH}"
--env-output "${monthly_publish_env}"
)
if [ -n "${AS_OF_DATE:-}" ]; then
monthly_args+=(--as-of "${AS_OF_DATE}")
fi
python scripts/check_monthly_snapshot_publish_window.py "${monthly_args[@]}"
# shellcheck disable=SC1090
source "${monthly_publish_env}"
;;
esac
fi
{
echo "should_publish=${should_publish}"
echo "publish_reason=${publish_reason}"
echo "snapshot_as_of=${snapshot_as_of}"
echo "month_end_trading_day=${month_end_trading_day}"
echo "calendar_source=${calendar_source}"
} >> "$GITHUB_OUTPUT"
if [ "${should_publish}" != "true" ]; then
cat > "${artifact_dir}/SKIPPED_MONTHLY_SNAPSHOT_PUBLISH.txt" <<EOF
profile=${PROFILE}
reason=${publish_reason}
snapshot_as_of=${snapshot_as_of}
month_end_trading_day=${month_end_trading_day}
calendar_source=${calendar_source}
EOF
echo "Skipping monthly snapshot publish: ${publish_reason}; snapshot_as_of=${snapshot_as_of}; month_end_trading_day=${month_end_trading_day}"
exit 0
fi
common_args=(--output-dir "${artifact_dir}")
if [ -n "${PRICES_PATH:-}" ]; then
common_args+=(--prices "${PRICES_PATH}")
fi
if [ -n "${UNIVERSE_PATH:-}" ] && [ "${PROFILE}" != "global_etf_rotation" ]; then
common_args+=(--universe "${UNIVERSE_PATH}")
fi
if [ -n "${AS_OF_DATE:-}" ]; then
common_args+=(--as-of "${AS_OF_DATE}")
fi
if [ -n "${CURRENT_HOLDINGS:-}" ] && [ "${PROFILE}" != "global_etf_rotation" ]; then
common_args+=(--current-holdings "${CURRENT_HOLDINGS}")
fi
if [ -n "${SOURCE_INPUT_MANIFEST_PATH:-}" ] && [ "${PROFILE}" != "global_etf_rotation" ]; then
common_args+=(--source-input-manifest "${SOURCE_INPUT_MANIFEST_PATH}")
fi
case "${PROFILE}" in
russell_top50_leader_rotation)
if [ -n "${MIN_ADV20_USD:-}" ]; then
common_args+=(--min-adv20-usd "${MIN_ADV20_USD}")
fi
if [ -n "${PORTFOLIO_TOTAL_EQUITY:-}" ]; then
common_args+=(--portfolio-total-equity "${PORTFOLIO_TOTAL_EQUITY}")
fi
python scripts/build_russell_top50_leader_rotation_snapshot.py "${common_args[@]}"
;;
global_etf_rotation)
python scripts/build_global_etf_rotation_snapshot.py \
"${common_args[@]}" \
--price-start "2022-01-01" \
--extra-history-symbols "QQQ,AAPL,MSFT,NVDA,AMZN,GOOGL,META,TSLA,AVGO"
;;
*)
echo "Unsupported profile: ${PROFILE}" >&2
exit 1
;;
esac
- name: Validate lifecycle market history
if: >-
env.PROFILE == 'global_etf_rotation' &&
steps.build.outputs.should_publish != 'false'
run: |
set -euo pipefail
test -s "${{ steps.build.outputs.artifact_dir }}/downloaded_price_history.csv"
- name: Publish or print GCS plan
if: steps.build.outputs.should_publish != 'false'
run: |
set -euo pipefail
if [ -z "${GCS_PREFIX:-}" ]; then
echo "gcs_prefix is not set; skipping GCS publish."
exit 0
fi
args=(
--profile "${PROFILE}"
--artifact-dir "${{ steps.build.outputs.artifact_dir }}"
--gcs-prefix "${GCS_PREFIX}"
)
if [ "${EXECUTE_PUBLISH}" = "true" ]; then
args+=(--execute)
args+=(--candidate-prefix "${GCS_PREFIX%/}/candidates/${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}")
fi
python scripts/publish_snapshot_artifacts.py "${args[@]}"
- name: Run Russell shadow cycle
if: >-
env.PROFILE == 'russell_top50_leader_rotation' &&
steps.build.outputs.should_publish != 'false'
run: |
set -euo pipefail
python scripts/run_russell_leader_rotation_shadow_cycle.py \
--feature-snapshot "${{ steps.build.outputs.artifact_dir }}/russell_top50_leader_rotation_feature_snapshot_latest.csv" \
--snapshot-as-of "${{ steps.build.outputs.snapshot_as_of }}" \
--output-dir "${{ steps.build.outputs.artifact_dir }}"
- name: Run Global ETF shadow cycle
if: >-
env.PROFILE == 'global_etf_rotation' &&
steps.build.outputs.should_publish != 'false'
run: |
set -euo pipefail
python scripts/run_global_etf_rotation_shadow_cycle.py \
--feature-snapshot "${{ steps.build.outputs.artifact_dir }}/global_etf_rotation_feature_snapshot_latest.csv" \
--snapshot-as-of "${{ steps.build.outputs.snapshot_as_of }}" \
--output-dir "${{ steps.build.outputs.artifact_dir }}"
- name: Upload generated artifacts
uses: actions/upload-artifact@v7
with:
name: ${{ steps.build.outputs.artifact_name }}
path: ${{ steps.build.outputs.artifact_dir }}
if-no-files-found: error
retention-days: 7
- name: Upload lifecycle market history
if: >-
env.PROFILE == 'global_etf_rotation' &&
steps.build.outputs.should_publish != 'false'
uses: actions/upload-artifact@v7
with:
name: us-equity-market-history-${{ github.run_id }}-${{ github.run_attempt }}
path: ${{ steps.build.outputs.artifact_dir }}/downloaded_price_history.csv
if-no-files-found: error
retention-days: 30
- name: Append job summary
run: |
set -euo pipefail
{
echo "## Snapshot artifact run"
echo
echo "- profile: \`${PROFILE}\`"
echo "- artifact_dir: \`${{ steps.build.outputs.artifact_dir }}\`"
echo "- use_sample_data: \`${USE_SAMPLE_DATA}\`"
echo "- source_input_manifest_path: \`${SOURCE_INPUT_MANIFEST_PATH:-<not set>}\`"
echo "- gcs_prefix: \`${GCS_PREFIX:-<not set>}\`"
echo "- execute_publish: \`${EXECUTE_PUBLISH}\`"
echo "- should_publish: \`${{ steps.build.outputs.should_publish }}\`"
echo "- publish_reason: \`${{ steps.build.outputs.publish_reason }}\`"
echo "- snapshot_as_of: \`${{ steps.build.outputs.snapshot_as_of }}\`"
echo "- month_end_trading_day: \`${{ steps.build.outputs.month_end_trading_day }}\`"
echo "- calendar_source: \`${{ steps.build.outputs.calendar_source }}\`"
echo
echo "Generated files:"
find "${{ steps.build.outputs.artifact_dir }}" -maxdepth 1 -type f -print | sort | sed 's/^/- /'
} >> "$GITHUB_STEP_SUMMARY"
rerun-scheduled-failure:
needs: build-and-publish
if: ${{ failure() && github.event_name == 'schedule' && fromJSON(github.run_attempt) < 2 }}
runs-on: ubuntu-latest
timeout-minutes: 60
permissions:
actions: write
contents: read
steps:
- name: Re-run failed jobs once
continue-on-error: true
env:
GH_TOKEN: ${{ github.token }}
run: |
set -euo pipefail
if ! gh run rerun "${GITHUB_RUN_ID}" --failed --repo "${GITHUB_REPOSITORY}"; then
echo "::warning::Failed to request rerun for ${GITHUB_RUN_ID}; the run may still be active."
fi