run_experiments.sh

#!/bin/bash

# ==============================================================================
# V2X QoS Large-Scale Simulation and Analytics Control Console
# ==============================================================================

export LC_ALL=C
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

# Initialize terminal signaling color codes
C_RESET="\033[0m"
C_BOLD="\033[1m"
C_INFO="\033[1;36m"
C_SUCCESS="\033[1;32m"
C_WARN="\033[1;33m"
C_ERROR="\033[1;31m"

# Print usage helper screen
print_usage() {
    echo -e "${C_BOLD}V2X QoS Large-Scale Simulation and Analytics Control Console${C_RESET}"
    echo -e "${C_INFO}Usage:${C_RESET} ./run_experiments.sh ${C_SUCCESS}[target]${C_RESET} ${C_WARN}[action]${C_RESET} [optional flags...]"
    echo -e ""
    echo -e "${C_BOLD}Targets:${C_RESET}"
    echo -e "  ${C_SUCCESS}unpatched${C_RESET}            Run tasks against the unpatched system workspace"
    echo -e "  ${C_SUCCESS}patched${C_RESET}              Run the recursion-depth-limited comparison workspace"
    echo -e "  ${C_SUCCESS}all${C_RESET}                  Sequentially run entire matrix for BOTH unpatched and patched"
    echo -e "  ${C_SUCCESS}python${C_RESET}               Orchestrate Python DRL agent scripts (venv encapsulated)"
    echo -e ""
    echo -e "${C_BOLD}Actions under C++ targets (unpatched / patched / all):${C_RESET}"
    echo -e "  ${C_WARN}--diagnose-flood${C_RESET}      Execute short flood region parsing delta diagnosis"
    echo -e "  ${C_WARN}--profile-amp${C_RESET}         Run full geometric size sweep tracking MTU capacity scaling"
    echo -e "  ${C_WARN}--build-dataset${C_RESET}       Generate and validate high-potency toxic exploit vectors"
    echo -e "  ${C_WARN}--simulate-all${C_RESET}        Execute matrix sweep dynamically configured by runtime filters/ranges"
    echo -e "  ${C_WARN}--train-rl${C_RESET}            Feed C++ trajectories to the online learner over the socket bridge"
    echo -e "  ${C_WARN}receive${C_RESET}               Start the C++ UDP receiver used on the edge node"
    echo -e "  ${C_WARN}send${C_RESET}                  Start the built-in C++ UDP sender (Python sender is used for paper runs)"
    echo -e ""
    echo -e "${C_BOLD}Actions under 'python' target:${C_RESET}"
    echo -e "  ${C_WARN}--train-online${C_RESET}       Start the configured DQN/PPO online learner (paper path: DQN)"
    echo -e "  ${C_WARN}--train-offline${C_RESET}      Run the optional offline dataset trainer (not the paper model path)"
    echo -e "  ${C_WARN}--deploy${C_RESET}             Start the legacy Python inference daemon"
    echo -e "  ${C_WARN}--verify-brain${C_RESET}       Audit brain checkpoints on baseline scenarios"
    echo -e "  ${C_WARN}--export-onnx${C_RESET}        Export trained PyTorch model weights to ONNX format"
    echo -e "  ${C_WARN}--verify-onnx${C_RESET}        Run numeric alignment check between PyTorch, ONNX, and C++"
    echo -e "  ${C_WARN}--plot${C_RESET}               Execute verification and plotting engine scripts"
    echo -e "  ${C_WARN}--clean${C_RESET}              Purge telemetry CSV logs and old window plots to prevent data pollution"
    echo -e "  ${C_WARN}--test${C_RESET}               Run python strategy consistency unit tests via pytest"
    echo -e "  ${C_INFO}  * Tip: Append -h/--help to any python action to view its specific parameters${C_RESET}"
    echo -e "    (e.g., ./run_experiments.sh python --train-online -h)"
    echo -e ""
    echo -e "${C_BOLD}Rate and safety modifiers:${C_RESET}"
    echo -e "  ${C_INFO}-r, --rates <\"rates\">${C_RESET}   Session-global pollution percentages (Default: \"1.0 5.0 10.0\")."
    echo -e "                          Python offline training additionally accepts the special value \"mix\"."
    echo -e "  ${C_INFO}-s, --disable-safety${C_RESET}  Disable C++ policy safety clamps (diagnostic use only)"
    echo -e ""
    echo -e "${C_BOLD}C++ Simulation Specific Modifiers:${C_RESET}"
    echo -e "  ${C_INFO}-c, --core, --data-core <id>${C_RESET}  Data-plane CPU core (Default: 2)"
    echo -e "  ${C_INFO}--control-core <id>${C_RESET}    ONNX worker CPU (Default: next non-zero core when possible)"
    echo -e "  ${C_INFO}-n, --no-taskset${C_RESET}      Disable explicit process/thread CPU affinity"
    echo -e "  ${C_INFO}-B, --baseline-only${C_RESET}   Execute ONLY Filter=OFF simulation steps (No mitigation)"
    echo -e "  ${C_INFO}-F, --filter-only${C_RESET}     Execute ONLY Filter=ON simulation steps (Mitigation active)"
    echo -e "  ${C_INFO}-m, --modes <\"modes\">${C_RESET}   Override default simulation scenario modes (Default: \"0 1 2\")."
    echo -e "                          Scenarios:"
    echo -e "                            0 = Uniform Random Attack (Malware dispersed randomly)"
    echo -e "                            1 = Single Pulse Attack (Sudden burst at 30%-50% window)"
    echo -e "                            2 = Periodic On-Off (5 waves of peak attack cycles)"
    echo -e "                            3 = Grand Mix Scenario (Dynamic hybrid mix for RL training)"
    echo -e "  ${C_INFO}-o, --onnx [path]${C_RESET}     Enable in-process ONNX model inference during simulation"
    echo -e "  ${C_INFO}-S, --static-100${C_RESET}      Force static 100% inspection for the filter path"
    echo -e "  ${C_INFO}-N, --packets <count>${C_RESET} Total packets per experiment (Default: 1000000)"
    echo -e "  ${C_INFO}-l, --lambda <pps>${C_RESET}    Aggregate packet arrival rate"
    echo -e "  ${C_INFO}-P, --port <port>${C_RESET}     UDP receiver/sender port (Default: 9999)"
    echo -e "  ${C_INFO}-ip, --dest-ip <addr>${C_RESET} UDP destination address"
    echo -e "  ${C_INFO}--zip${C_RESET}                 Pair modes and rates by index for online-training trajectories"
    echo -e "  ${C_INFO}--sequence-file <path>${C_RESET} Read online-training mode/rate pairs from a file"
    echo -e "  ${C_INFO}--dry-run${C_RESET}             Print resolved training commands without executing them"
    echo -e "  ${C_INFO}--onnx-fixed-policy${C_RESET}    Replace ORT inference with a fixed safe policy for diagnosis"
    echo -e "  ${C_INFO}--onnx-diagnostics${C_RESET}     Enable asynchronous ONNX timing/mailbox diagnostics"
    echo -e "  ${C_INFO}--data-plane-diagnostics${C_RESET} Enable filter/parser timing diagnostics"
    echo -e ""
    echo -e "${C_BOLD}Python DRL Specific Modifiers:${C_RESET}"
    echo -e "  ${C_INFO}--frame-stack <k>${C_RESET}    Specify frame stacking size (K=1 is stateless; default: from agent.yaml)"
    echo -e "  ${C_INFO}--fresh${C_RESET}             Bypass loading existing PyTorch weights and train from scratch"
    echo -e ""
    echo -e "${C_BOLD}Examples:${C_RESET}"
    echo -e "  ./run_experiments.sh ${C_SUCCESS}unpatched${C_RESET} ${C_WARN}--simulate-all${C_RESET} -m \"0 1\" -r \"1.0 5.0\"      \033[90m# Run specific C++ sweep\033[0m"
    echo -e "  ./run_experiments.sh ${C_SUCCESS}python${C_RESET} ${C_WARN}--train-online${C_RESET} --algorithm dqn              \033[90m# Paper training path\033[0m"
    echo -e "  ./run_experiments.sh ${C_SUCCESS}unpatched${C_RESET} ${C_WARN}--simulate-all${C_RESET} -F -o -n                   \033[90m# Run v7 ONNX sweep\033[0m"
    echo -e "  ./run_experiments.sh ${C_SUCCESS}unpatched${C_RESET} ${C_WARN}receive${C_RESET} -P 9999 -o                        \033[90m# Raspberry Pi receiver\033[0m"
    exit "${1:-1}"
}

if [ "${1:-}" = "-h" ] || [ "${1:-}" = "--help" ]; then
    print_usage 0
fi

# ── EARLY ROUTING: Handle Python target to prevent argument swallowing ──────
if [ "$1" = "python" ]; then
    TARGET="python"
    ACTION=$2
    if [ -z "$ACTION" ]; then
        print_usage
    fi
    shift 2

    # Map the action string to the targeted local python script
    case "$ACTION" in
        --train-online)
            SCRIPT="scripts/train_online.py"
            ;;
        --train-offline)
            SCRIPT="scripts/train_offline.py"
            ;;
        --deploy)
            SCRIPT="scripts/serve_agent.py"
            ;;
        --verify-brain)
            SCRIPT="scripts/verify_brain.py"
            ;;
        --export-onnx)
            SCRIPT="scripts/export_onnx.py"
            ;;
        --verify-onnx)
            SCRIPT="../verify_equivalence.py"
            ;;
        --plot)
            # Plot engine is in the tools folder
            SCRIPT="../plot_engine.py"
            ;;
        --clean)
            echo -e "${C_INFO}[*] Purging stale window telemetry CSV logs and plots...${C_RESET}"
            rm -f "${ROOT_DIR}/outputs/rl_env/unpatched"/*.csv
            rm -f "${ROOT_DIR}/outputs/rl_env/patched"/*.csv
            rm -f "${ROOT_DIR}/checkpoints"/online_training_telemetry.csv
            rm -rf "${ROOT_DIR}/outputs/plots/qos/window"/*
            echo -e "${C_SUCCESS}[SUCCESS] Purge completed safely. Stale CSV/plot data cleared.${C_RESET}"
            exit 0
            ;;
        --test|--run-tests)
            # Unified testing entry point executing consistency checks via pytest
            PYTHON_EXEC="${ROOT_DIR}/tools/rl_bridge/venv/bin/python3"
            if [ ! -f "$PYTHON_EXEC" ]; then PYTHON_EXEC="python3"; fi
            
            if "$PYTHON_EXEC" -c "import pytest" &>/dev/null; then
                echo -e "${C_INFO}[*] Running consistency tests via pytest...${C_RESET}"
                exec "$PYTHON_EXEC" -m pytest "${ROOT_DIR}/tools/rl_bridge/tests/test_consistency.py" "$@"
            else
                echo -e "${C_ERROR}[ERROR] pytest is not installed in the virtual environment.${C_RESET}"
                echo -e "${C_WARN}[SUGGESTION] Please install it using: tools/rl_bridge/venv/bin/pip install pytest${C_RESET}"
                exit 1
            fi
            ;;
        *)
            print_usage
            ;;
    esac

    # Resolve local python executable inside virtualenv
    PYTHON_EXEC="${ROOT_DIR}/tools/rl_bridge/venv/bin/python3"
    if [ ! -f "$PYTHON_EXEC" ]; then PYTHON_EXEC="python3"; fi

    # Launch python script and pass remaining arguments untouched
    if [ "$ACTION" = "--train-online" ]; then
        exec "$PYTHON_EXEC" "${ROOT_DIR}/tools/rl_bridge/src/main.py" --mode online "$@"
    elif [ "$ACTION" = "--train-offline" ]; then
        exec "$PYTHON_EXEC" "${ROOT_DIR}/tools/rl_bridge/src/main.py" --mode offline "$@"
    else
        exec "$PYTHON_EXEC" "${ROOT_DIR}/tools/rl_bridge/${SCRIPT}" "$@"
    fi
fi

# ── C++ SIMULATION TARGET ARGUMENT PARSING ───────────────────────────────────
TARGET=$1
ACTION=$2

if [ -z "$TARGET" ] || { [ "$TARGET" != "unpatched" ] && [ "$TARGET" != "patched" ] && [ "$TARGET" != "all" ]; } || [ -z "$ACTION" ]; then
    print_usage
fi

shift 2

# Initialize baseline default states and matrix parameters
PIN_CORE=${PIN_CORE:-2}
CONTROL_CORE=""
USE_TASKSET=true
TOTAL_PACKETS=1000000
RUN_FILTER_OFF=true
RUN_FILTER_ON=true
RUN_ONNX=false
ONNX_MODEL_PATH=""
DISABLE_SAFETY=false
RUN_TRACE=false
ONNX_FIXED_POLICY_DIAGNOSTIC=false
ONNX_DIAGNOSTICS_OVERRIDE=""
DATA_PLANE_DIAGNOSTICS_OVERRIDE=""
LAMBDA_PPS=""


# Chi-AN : experiment data flow control  parameters 

ZIP_MODE=false
SEQUENCE_FILE=""
DRY_RUN=false

# Default global state machine mode and pollution density spectrum arrays
DEFAULT_MODES=(0 1 2)
TARGET_MODES=("${DEFAULT_MODES[@]}")

DEFAULT_RATES=(1.0 5.0 10.0)
POLLUTION_RATES=("${DEFAULT_RATES[@]}")

# Loop through C++ specific arguments
while [[ $# -gt 0 ]]; do
    case "$1" in
        -t|-T|--trace)
            RUN_TRACE=true
            shift
            ;;
        -c|--core|--data-core)
            if [[ -n "$2" && "$2" =~ ^[0-9]+$ ]]; then
                PIN_CORE="$2"
                shift 2
            else
                echo -e "${C_ERROR}[ERROR] -c/--core demands a valid numeric CPU index value.${C_RESET}"
                exit 1
            fi
            ;;
        --control-core)
            if [[ -n "$2" && "$2" =~ ^[0-9]+$ ]]; then
                CONTROL_CORE="$2"
                shift 2
            else
                echo -e "${C_ERROR}[ERROR] --control-core demands a valid numeric CPU index value.${C_RESET}"
                exit 1
            fi
            ;;
        -n|--no-taskset|--no-pin)
            USE_TASKSET=false
            shift
            ;;
        -B|--baseline-only|--no-filter-only)
            RUN_FILTER_OFF=true
            RUN_FILTER_ON=false
            shift
            ;;
        -F|--filter-only)
            RUN_FILTER_OFF=false
            RUN_FILTER_ON=true
            shift
            ;;
        -m|--modes)
            if [[ -n "$2" ]]; then
                IFS=' ' read -r -a TARGET_MODES <<< "$2"
                shift 2
            else
                echo -e "${C_ERROR}[ERROR] -m/--modes demands a space-separated string of integers (e.g. \"0 1\").${C_RESET}"
                exit 1
            fi
            ;;
        -l|--lambda)
            if [[ -n "$2" && ! "$2" =~ ^- ]]; then
                LAMBDA_PPS="$2"
                shift 2
            else
                LAMBDA_PPS="3000"
                shift 1
            fi
            ;;
        -r|--rates|--rate)
            if [[ -n "$2" ]]; then
                # Safe Boundary Check: "mix" is invalid for C++ targets
                if [[ "$2" == *"mix"* ]]; then
                    echo -e "${C_ERROR}[ERROR] C++ simulation targets only accept numeric rates (e.g., \"1.0 5.0\").${C_RESET}"
                    echo -e "${C_WARN}[NOTICE] \"mix\" is reserved for python --train-offline offline dataset blending.${C_RESET}"
                    exit 1
                fi
                IFS=' ' read -r -a POLLUTION_RATES <<< "$2"
                shift 2
            else
                echo -e "${C_ERROR}[ERROR] -r/--rates demands a space-separated string of numbers.${C_RESET}"
                exit 1
            fi
            ;;
        -o|--onnx)
            RUN_ONNX=true
            if [[ -n "$2" && ! "$2" =~ ^- ]]; then
                ONNX_MODEL_PATH="$2"
                # If only a filename is provided (no slashes), search in the default checkpoints directory
                if [[ "$ONNX_MODEL_PATH" != *"/"* ]]; then
                    ONNX_MODEL_PATH="${ROOT_DIR}/checkpoints/${ONNX_MODEL_PATH}"
                fi
                shift 2
            else
                # No explicit override: native C++ resolves model_path from agent.yaml.
                ONNX_MODEL_PATH=""
                shift
            fi
            ;;
        --onnx-fixed-policy)
            ONNX_FIXED_POLICY_DIAGNOSTIC=true
            shift
            ;;
        --onnx-diagnostics)
            ONNX_DIAGNOSTICS_OVERRIDE="enable"
            shift
            ;;
        --no-onnx-diagnostics)
            ONNX_DIAGNOSTICS_OVERRIDE="disable"
            shift
            ;;
        --data-plane-diagnostics)
            DATA_PLANE_DIAGNOSTICS_OVERRIDE="enable"
            shift
            ;;
        --no-data-plane-diagnostics)
            DATA_PLANE_DIAGNOSTICS_OVERRIDE="disable"
            shift
            ;;
        -s|--disable-safety)
            DISABLE_SAFETY=true
            shift
            ;;
        -S|--static-100|--full-100|--static-filter)
            RUN_STATIC_100=true
            shift
            ;;
        -P|--port)
            UDP_PORT="$2"
            shift 2
            ;;
        -ip|--dest-ip|--ip)
            DEST_IP="$2"
            shift 2
            ;;
        #===================================Chi-AN: new command for training data flow =======================================
        --zip|--paired)
            ZIP_MODE=true
            shift
            ;;

        --sequence-file|--seq-file)
            if [[ -n "$2" && ! "$2" =~ ^- ]]; then
                SEQUENCE_FILE="$2"
                # If path has no slash, default to tools/trainingConfigs directory
                if [[ "$SEQUENCE_FILE" != *"/"* ]]; then
                    SEQUENCE_FILE="${ROOT_DIR}/tools/trainingConfigs/${SEQUENCE_FILE}"
                fi
                shift 2
            else
                echo -e "${C_ERROR}[ERROR] --sequence-file demands a file path.${C_RESET}"
                exit 1
            fi
            ;;

        --dry-run)
            DRY_RUN=true
            shift
            ;;

        -l|--lambda)
            if [[ -n "$2" && "$2" =~ ^[0-9]+(\.[0-9]+)?$ ]]; then
                LAMBDA_PPS="$2"
                shift 2
            else
                echo -e "${C_ERROR}[ERROR] -l/--lambda demands a numeric rate.${C_RESET}"
                exit 1
            fi
            ;;
        -N|--packets)
            if [[ -n "$2" && "$2" =~ ^[0-9]+$ ]]; then
                TOTAL_PACKETS="$2"
                shift 2
            else
                echo -e "${C_ERROR}[ERROR] -N/--packets demands a positive integer.${C_RESET}"
                exit 1
            fi
            ;;
        *)
            echo -e "${C_ERROR}[ERROR] Unknown option: $1${C_RESET}"
            exit 1
            ;;
    esac
done

# Verify mutual exclusivity: ONNX mode (-o/--onnx) demands the FSM filter to be active
if [ "$RUN_ONNX" = true ] && [ "$RUN_FILTER_ON" = false ]; then
    echo -e "${C_ERROR}[ERROR] Conflict: ONNX mode (-o/--onnx) and Baseline-Only (-B) are mutually exclusive.${C_RESET}"
    echo -e "${C_WARN}[NOTICE] ONNX inference requires the pre-filter to be active, but -B disables filtering.${C_RESET}"
    exit 1
fi

# If ONNX mode is enabled, verify that the ONNX model file exists before execution
if [ "$RUN_ONNX" = true ] && [ -n "$ONNX_MODEL_PATH" ]; then
    if [ ! -f "$ONNX_MODEL_PATH" ]; then
        echo -e "${C_ERROR}[ERROR] ONNX model file not found at: ${ONNX_MODEL_PATH}${C_RESET}"
        echo -e "${C_WARN}[NOTICE] Please make sure to compile and export the DRL model to ONNX format first.${C_RESET}"
        echo -e "${C_INFO}[INFO] Gracefully exiting.${C_RESET}"
        exit 0
    fi
fi

# Helper execution engine bypassing taskset if USE_TASKSET is set to false
execute_cmd() {
    if [ "$USE_TASKSET" = true ]; then
        local num_cores
        num_cores=$(nproc)
        local pin_core_wrapped=$(( PIN_CORE % num_cores ))
        if [ "$RUN_ONNX" = true ]; then
            local onnx_core
            if [ -n "$CONTROL_CORE" ]; then
                onnx_core=$(( CONTROL_CORE % num_cores ))
            elif [ "$num_cores" -le 1 ]; then
                onnx_core=0
            else
                onnx_core=$(( (pin_core_wrapped + 1) % num_cores ))
                # Preserve a distinct core on two-core systems. With 3+ CPUs,
                # avoid CPU 0 because it commonly handles Linux housekeeping.
                if [ "$onnx_core" -eq 0 ] && [ "$num_cores" -gt 2 ]; then
                    onnx_core=1
                fi
            fi
            taskset -c "${pin_core_wrapped},${onnx_core}" "$@"
        else
            taskset -c "$pin_core_wrapped" "$@"
        fi
    else
        "$@"
    fi
    local status=$?
    if [ $status -ne 0 ]; then
        # If the command failed or was terminated by Ctrl+C (exit status 130)
        exit $status
    fi
}

# ===============Chi-AN: new function for training data flow control ==========================================
run_training_pair() {
    local tgt="$1"
    local exec_bin="$2"
    local mode="$3"
    local rate="$4"

    echo -e "${C_INFO}[EXEC] RL trajectory: target=${tgt}, mode=${mode}, rate=${rate}%, packets=${TOTAL_PACKETS}${C_RESET}"

    local train_args=("-t" "$TOTAL_PACKETS" "-p" "$rate" "-m" "$mode" "--rl")
    if [ -n "$LAMBDA_PPS" ]; then
        train_args+=("--lambda" "$LAMBDA_PPS")
    fi

    if [ "$DISABLE_SAFETY" = true ]; then
        train_args+=("--disable-safety")
    fi

    local status=0
    if [ "$DRY_RUN" = true ]; then
        echo -e "${C_WARN}[DRY-RUN] $exec_bin ${train_args[*]}${C_RESET}"
    else
        execute_cmd "$exec_bin" "${train_args[@]}"
        status=$?
    fi

    echo "----------------------------------------------------------------------"
    return $status
}

# Resolve lock status representation for console logs
get_core_info() {
    if [ "$USE_TASKSET" = true ]; then
        local num_cores
        num_cores=$(nproc)
        local pin_core_wrapped=$(( PIN_CORE % num_cores ))
        if [ "$RUN_ONNX" = true ]; then
            local onnx_core
            if [ -n "$CONTROL_CORE" ]; then
                onnx_core=$(( CONTROL_CORE % num_cores ))
            elif [ "$num_cores" -le 1 ]; then
                onnx_core=0
            else
                onnx_core=$(( (pin_core_wrapped + 1) % num_cores ))
                if [ "$onnx_core" -eq 0 ] && [ "$num_cores" -gt 2 ]; then
                    onnx_core=1
                fi
            fi
            echo "Cores: ${pin_core_wrapped},${onnx_core}"
        else
            echo "Core: ${pin_core_wrapped}"
        fi
    else
        echo "Core: Dynamic (OS-scheduled)"
    fi
}

# ── NEW: Dedicated Modular Pipeline Function for RL Training ────────────────
execute_rl_training() {
    local tgt=$1
    local exec_bin="${ROOT_DIR}/vanetza_${tgt}/build/bin/qos-harness"

    if [ ! -f "$exec_bin" ]; then
        echo -e "${C_ERROR}[FATAL] Executable target kernel missing at ${exec_bin}.${C_RESET}"
        echo -e "${C_ERROR}[FATAL] Run './manage_build.sh ${tgt}' first to compile binary map.${C_RESET}"
        exit 1
    fi

    mkdir -p "${ROOT_DIR}/outputs/rl_env"

    echo -e "${C_INFO}[STAGE] RL Training | Target=${tgt} | Packets=${TOTAL_PACKETS} | Core=$(get_core_info)${C_RESET}"

    # Priority 1: read explicit sequence file
    if [ -n "$SEQUENCE_FILE" ]; then
        if [ ! -f "$SEQUENCE_FILE" ]; then
            echo -e "${C_ERROR}[ERROR] Sequence file not found: ${SEQUENCE_FILE}${C_RESET}"
            exit 1
        fi

        echo -e "${C_INFO}[MODE] Using sequence file: ${SEQUENCE_FILE}${C_RESET}"

        local line_no=0
        while read -r mode rate extra; do
            line_no=$((line_no + 1))

            # Skip empty lines and comments
            [[ -z "$mode" ]] && continue
            [[ "$mode" =~ ^# ]] && continue

            if [[ -z "$rate" ]]; then
                echo -e "${C_ERROR}[ERROR] Invalid sequence line ${line_no}: missing rate.${C_RESET}"
                exit 1
            fi

            if [[ -n "$extra" ]]; then
                echo -e "${C_WARN}[WARN] Extra tokens ignored at line ${line_no}: ${extra}${C_RESET}"
            fi

            run_training_pair "$tgt" "$exec_bin" "$mode" "$rate"
            local exit_code=$?
            if [ $exit_code -eq 130 ] || [ $exit_code -eq 137 ] || [ $exit_code -eq 143 ]; then
                echo -e "${C_WARN}[NOTICE] Sequence training loop aborted by user (SIGINT/SIGTERM).${C_RESET}"
                exit 130
            fi
        done < "$SEQUENCE_FILE"

        return
    fi

    # Priority 2: paired arrays
    if [ "$ZIP_MODE" = true ]; then
        echo -e "${C_INFO}[MODE] Using zipped mode/rate arrays.${C_RESET}"

        if [ "${#TARGET_MODES[@]}" -ne "${#POLLUTION_RATES[@]}" ]; then
            echo -e "${C_ERROR}[ERROR] --zip requires the number of modes and rates to match.${C_RESET}"
            echo -e "${C_ERROR}[ERROR] modes=${#TARGET_MODES[@]}, rates=${#POLLUTION_RATES[@]}${C_RESET}"
            exit 1
        fi

        for i in "${!TARGET_MODES[@]}"; do
            run_training_pair "$tgt" "$exec_bin" "${TARGET_MODES[$i]}" "${POLLUTION_RATES[$i]}"
            local exit_code=$?
            if [ $exit_code -eq 130 ] || [ $exit_code -eq 137 ] || [ $exit_code -eq 143 ]; then
                echo -e "${C_WARN}[NOTICE] Paired training loop aborted by user (SIGINT/SIGTERM).${C_RESET}"
                exit 130
            fi
        done

        return
    fi

    # Priority 3: original matrix behavior
    echo -e "${C_INFO}[MODE] Using matrix sweep: modes x rates.${C_RESET}"

    for mode in "${TARGET_MODES[@]}"; do
        for rate in "${POLLUTION_RATES[@]}"; do
            run_training_pair "$tgt" "$exec_bin" "$mode" "$rate"
            local exit_code=$?
            if [ $exit_code -eq 130 ] || [ $exit_code -eq 137 ] || [ $exit_code -eq 143 ]; then
                echo -e "${C_WARN}[NOTICE] Matrix sweep training loop aborted by user (SIGINT/SIGTERM).${C_RESET}"
                exit 130
            fi
        done
    done
}

execute_matrix_sweep() {
    local tgt=$1
    local exec_bin="${ROOT_DIR}/vanetza_${tgt}/build/bin/qos-harness"

    if [ ! -f "$exec_bin" ]; then
        echo -e "${C_ERROR}[FATAL] Executable target kernel missing at ${exec_bin}.${C_RESET}"
        echo -e "${C_ERROR}[FATAL] Run './manage_build.sh ${tgt}' first to compile binary map.${C_RESET}"
        exit 1
    fi

    mkdir -p "${ROOT_DIR}/outputs/csv_raw/${tgt}"

    # Dynamic target mode list mapped from command arguments
    for mode in "${TARGET_MODES[@]}"; do
        for rate in "${POLLUTION_RATES[@]}"; do
            
            # Conditionally execute the unhardened raw baseline telemetry flow node
            if [ "$RUN_FILTER_OFF" = true ]; then
                echo -e "${C_INFO}[STAGE] Node Init: Target=${tgt} | Rate=${rate}% | Mode=${mode} | Filter=OFF | ONNX=OFF ($(get_core_info))${C_RESET}"
                local sweep_args_off=("-t" "$TOTAL_PACKETS" "-p" "$rate" "-m" "$mode")
                if [ -n "$LAMBDA_PPS" ]; then
                    sweep_args_off+=("--lambda" "$LAMBDA_PPS")
                fi
                execute_cmd "$exec_bin" "${sweep_args_off[@]}"
            fi

            # Conditionally execute the circuit-breaker state-machine hardened flow node
            if [ "$RUN_FILTER_ON" = true ]; then
                local onnx_status="OFF"; if [ "$RUN_ONNX" = true ]; then onnx_status="ON"; fi
                echo -e "${C_INFO}[STAGE] Node Init: Target=${tgt} | Rate=${rate}% | Mode=${mode} | Filter=ON  | ONNX=${onnx_status} ($(get_core_info))${C_RESET}"
                local sweep_args=("-t" "$TOTAL_PACKETS" "-p" "$rate" "-m" "$mode" "-f")
                if [ -n "$LAMBDA_PPS" ]; then
                    sweep_args+=("--lambda" "$LAMBDA_PPS")
                fi
                if [ "$RUN_ONNX" = true ]; then
            sweep_args+=("--onnx")
            if [ -n "$ONNX_MODEL_PATH" ]; then sweep_args+=("$ONNX_MODEL_PATH"); fi
                fi
                if [ "$RUN_STATIC_100" = true ]; then
                    sweep_args+=("--static-filter" "1.0")
                fi
                if [ "$DISABLE_SAFETY" = true ]; then
                    sweep_args+=("--disable-safety")
                fi
                if [ "$RUN_TRACE" = true ]; then
                    sweep_args+=("--trace")
                fi
                execute_cmd "$exec_bin" "${sweep_args[@]}"
            fi
            
            if [ "$RUN_FILTER_OFF" = true ] || [ "$RUN_FILTER_ON" = true ]; then
                echo "----------------------------------------------------------------------"
            fi
        done
    done
}

case "$ACTION" in
    --diagnose-flood|--profile-amp|--build-dataset)
        execute_single_action() {
            local tgt=$1
            local bin="${ROOT_DIR}/vanetza_${tgt}/build/bin/qos-harness"
            
            if [ ! -f "$bin" ]; then
                echo -e "${C_ERROR}[FATAL] Executable target kernel missing at ${bin}.${C_RESET}"
                echo -e "${C_ERROR}[FATAL] Run './manage_build.sh ${tgt}' first to compile binary map.${C_RESET}"
                exit 1
            fi
            
            echo -e "${C_INFO}[INFO] Triggering dedicated framework routine: ${ACTION} on target [${tgt}] ($(get_core_info))${C_RESET}"
            execute_cmd "$bin" "$ACTION"
            
            # Post-execution isolation router to prevent cross-over overwrites
            local target_csv_dir="${ROOT_DIR}/outputs/csv_raw/${tgt}"
            mkdir -p "$target_csv_dir"
            
            # Intercept and relocate generated profile data from shared or local drop zones
            if [ -f "${ROOT_DIR}/outputs/csv_raw/amplification_profile.csv" ]; then
                mv "${ROOT_DIR}/outputs/csv_raw/amplification_profile.csv" "${target_csv_dir}/"
                echo -e "${C_SUCCESS}[SUCCESS] Dynamic isolation complete: Relocated root profile to ${tgt}/ layout.${C_RESET}"
            elif [ -f "${ROOT_DIR}/vanetza_${tgt}/tools/qos-harness/csv_data/amplification_profile.csv" ]; then
                mv "${ROOT_DIR}/vanetza_${tgt}/tools/qos-harness/csv_data/amplification_profile.csv" "${target_csv_dir}/"
                echo -e "${C_SUCCESS}[SUCCESS] Dynamic isolation complete: Relocated localized profile to ${tgt}/ layout.${C_RESET}"
            fi
        }

        # Upgraded to natively support sequential automation sweeps via the 'all' keyword
        if [ "$TARGET" == "all" ]; then
            execute_single_action "unpatched"
            execute_single_action "patched"
        else
            execute_single_action "$TARGET"
        fi
        ;;
        
    --simulate-all)
        if [ "$TARGET" == "all" ]; then
            execute_matrix_sweep "unpatched"
            execute_matrix_sweep "patched"
        else
            execute_matrix_sweep "$TARGET"
        fi
        echo -e "${C_SUCCESS}[SUCCESS] Dynamic matrix sweep executed successfully. Data converged.${C_RESET}"
        ;;

    receive|--receive|listen|--listen)
        execute_receive() {
            local tgt=$1
            local bin="${ROOT_DIR}/vanetza_${tgt}/build/bin/qos-harness"
            if [ ! -f "$bin" ]; then
                echo -e "${C_ERROR}[FATAL] Executable kernel missing at ${bin}.${C_RESET}"
                echo -e "${C_ERROR}[FATAL] Run './manage_build.sh ${tgt}' first to compile binary map.${C_RESET}"
                exit 1
            fi
            echo -e "${C_INFO}[RECEIVER] Starting UDP Daemon on target [${tgt}] on port ${UDP_PORT:-9999}...${C_RESET}"
            local receiver_args=("--receive-udp" "${UDP_PORT:-9999}")
            if [ "$USE_TASKSET" = true ]; then
                local num_cores
                num_cores=$(nproc)
                local data_core=$(( PIN_CORE % num_cores ))
                local control_core
                if [ -n "$CONTROL_CORE" ]; then
                    control_core=$(( CONTROL_CORE % num_cores ))
                elif [ "$num_cores" -le 1 ]; then
                    control_core=0
                else
                    control_core=$(( (data_core + 1) % num_cores ))
                    if [ "$control_core" -eq 0 ] && [ "$num_cores" -gt 2 ]; then
                        control_core=1
                    fi
                fi
                receiver_args+=("--data-core" "$data_core" "--control-core" "$control_core")
            else
                receiver_args+=("--no-affinity")
            fi
            if [ "$RUN_ONNX" = true ]; then
                receiver_args+=("--onnx")
                if [ -n "$ONNX_MODEL_PATH" ]; then
                    receiver_args+=("$ONNX_MODEL_PATH")
                fi
            fi
            if [ "$ONNX_FIXED_POLICY_DIAGNOSTIC" = true ]; then
                receiver_args+=("--onnx-fixed-policy")
            fi
            if [ "$ONNX_DIAGNOSTICS_OVERRIDE" = "enable" ]; then
                receiver_args+=("--onnx-diagnostics")
            elif [ "$ONNX_DIAGNOSTICS_OVERRIDE" = "disable" ]; then
                receiver_args+=("--no-onnx-diagnostics")
            fi
            if [ "$DATA_PLANE_DIAGNOSTICS_OVERRIDE" = "enable" ]; then
                receiver_args+=("--data-plane-diagnostics")
            elif [ "$DATA_PLANE_DIAGNOSTICS_OVERRIDE" = "disable" ]; then
                receiver_args+=("--no-data-plane-diagnostics")
            fi
            execute_cmd "$bin" "${receiver_args[@]}"
        }
        if [ "$TARGET" == "all" ]; then
            execute_receive "unpatched"
            execute_receive "patched"
        else
            execute_receive "$TARGET"
        fi
        ;;

    send|--send|transmit|--transmit)
        execute_send() {
            local tgt=$1
            local bin="${ROOT_DIR}/vanetza_${tgt}/build/bin/qos-harness"
            if [ ! -f "$bin" ]; then
                echo -e "${C_ERROR}[FATAL] Executable kernel missing at ${bin}.${C_RESET}"
                echo -e "${C_ERROR}[FATAL] Run './manage_build.sh ${tgt}' first to compile binary map.${C_RESET}"
                exit 1
            fi
            local modes_str="${TARGET_MODES[*]}"
            local rates_str="${POLLUTION_RATES[*]}"
            local send_args=("--send-udp" "${DEST_IP:-127.0.0.1}" "--port" "${UDP_PORT:-9999}" "--modes" "$modes_str" "--rates" "$rates_str" "-t" "$TOTAL_PACKETS")
            if [ -n "$LAMBDA_PPS" ]; then
                send_args+=("--lambda" "$LAMBDA_PPS")
            fi
            if [ "$RUN_STATIC_100" = true ]; then
                send_args+=("--static-filter" "1.0")
            elif [ "$RUN_FILTER_ON" = true ]; then
                send_args+=("-f")
            fi
            echo -e "${C_INFO}[SENDER] Streaming UDP batch sessions to ${DEST_IP:-127.0.0.1}:${UDP_PORT:-9999} for target [${tgt}]...${C_RESET}"
            execute_cmd "$bin" "${send_args[@]}"
        }
        if [ "$TARGET" == "all" ]; then
            execute_send "unpatched"
            execute_send "patched"
        else
            execute_send "$TARGET"
        fi
        ;;

    --train-rl)
        if [ "$TARGET" == "all" ]; then
            execute_rl_training "unpatched"
            execute_rl_training "patched"
        else
            execute_rl_training "$TARGET"
        fi
        echo -e "${C_SUCCESS}[SUCCESS] Interactive RL training pipeline execution complete. Telemetry converged.${C_RESET}"
        ;;

    --custom)
        if [ "$TARGET" == "all" ]; then
            echo -e "${C_ERROR}[ERROR] Custom actions cannot accept global target routing.${C_RESET}"
            exit 1
        fi
        
        CUSTOM_ARGS=("$@")
        if [ "$RUN_ONNX" = true ]; then
            CUSTOM_ARGS+=("--onnx")
            if [ -n "$ONNX_MODEL_PATH" ]; then CUSTOM_ARGS+=("$ONNX_MODEL_PATH"); fi
        fi
        if [ "$DISABLE_SAFETY" = true ]; then
            CUSTOM_ARGS+=("--disable-safety")
        fi
        
        echo -e "${C_INFO}[INFO] Invoking custom framework map... Args: ${CUSTOM_ARGS[*]}${C_RESET}"
        EXEC_BIN="${ROOT_DIR}/vanetza_${TARGET}/build/bin/qos-harness"
        execute_cmd "$EXEC_BIN" "${CUSTOM_ARGS[@]}"
        ;;
        
    *)
        print_usage
        ;;
esac