diff --git a/cse-dr/PowerCalcsChatbot.Rmd b/cse-dr/PowerCalcsChatbot.Rmd new file mode 100644 index 0000000..7be8a72 --- /dev/null +++ b/cse-dr/PowerCalcsChatbot.Rmd @@ -0,0 +1,521 @@ +--- +title: "Power Calculations: AI Chatbot Component — ESI en Valores RCT" +author: "Tomás Buitrago | Development Innovation Lab" +date: "`r format(Sys.Date(), '%B %d, %Y')`" +output: + html_document: + toc: true + toc_float: + collapsed: false + toc_depth: 3 + theme: flatly + highlight: tango + code_folding: show + number_sections: true +--- + +```{r setup, include=FALSE} +knitr::opts_chunk$set( + echo = TRUE, + warning = FALSE, + message = FALSE, + fig.width = 9, + fig.height = 5.5, + fig.align = "center" +) +``` + +```{r libraries} +# ── Core packages ───────────────────────────────────────────────────────────── +library(tidyverse) # data wrangling and ggplot2 +library(pwr) # analytical power functions (pwr.t.test, pwr.2p.test) +library(knitr) # kable for formatted tables +library(kableExtra) # extended table styling +library(sandwich) # cluster-robust variance estimation (vcovCL) +library(lmtest) # coeftest() with heteroskedasticity/cluster-robust SEs + +theme_set( + theme_minimal(base_size = 12) + + theme( + plot.title = element_text(face = "bold"), + plot.subtitle = element_text(color = "grey40"), + panel.grid.minor = element_blank() + ) +) +``` + + +# Study Design + +**Two-stage randomisation:** + +**Stage 2 — Individual student-level (chatbot):** All students in the 450 treatment schools are individually randomized 50/50 to chatbot access vs. no chatbot, stratified by school CSE arm. 90 surveyed students per school (30 per grade × 3 grades), yields 40,500 students in the chatbot randomization. + + +# Key Parameters + +```{r parameters} +# Study design +n_schools_treatment <- 450 # total treatment schools +n_schools_per_arm <- 225 # per school-level arm (Regular CSE / CSE+Debiasing) +n_students_per_school <- 90 # surveyed per school (30/grade × 3 grades) + +# Chatbot randomization (individual-level 50/50 within each treatment school) +n_chatbot_total <- n_schools_treatment * n_students_per_school # 40,500 +n_per_chatbot_arm <- n_chatbot_total / 2 # 20,250 +n_per_cell_2x2 <- n_chatbot_total / 4 # 10,125 + +# Girls-only subsamples (assume 50% female) +n_girls_per_arm <- n_per_chatbot_arm / 2 # 10,125 +n_girls_per_cell <- n_per_cell_2x2 / 2 # 5,062 + +# Statistical conventions +alpha <- 0.05 +power_target <- 0.80 +z_alpha <- qnorm(1 - alpha / 2) # 1.960 +z_beta <- qnorm(power_target) # 0.842 + +# Intra-cluster correlation coefficients (ICC) +# ICC = share of total variance that lies between schools. + +icc_pregnancy <- 0.050 # ENHOGAR national survey +icc_overconfidence <- 0.065 # 2024 pilot study (N = 800 students) +icc_test_scores <- 0.200 # Pruebas Nacionales administrative data +icc_depression <- 0.100 # school-level health literature; pilot study +icc_knowledge <- 0.100 # within-school component for knowledge/behaviour index + +# Outcome-specific baseline parameters +# Binary outcomes: pregnancy rate +p0_preg_1yr <- 0.056 # annual hazard rate, girls 15–19 (WB WDI; Dupas 2018 SD) +p0_preg_4yr <- 0.168 # four-year cumulative rate + +# Continuous outcomes: overconfidence score (0–100 scale) +mean_overconf <- 10.4 # mean from 2024 pilot +sd_overconf <- 27.7 # SD from 2024 pilot + +# Standardized or Bernoulli outcomes (normalized SD = 1 or 0.5) +sd_test_scores <- 1.0 # Pruebas Nacionales, normalized +sd_depression <- 1.0 # standardized index +sd_knowledge <- 0.5 # Bernoulli SD at p0 = 0.50 +p0_knowledge <- 0.50 # baseline knowledge/behaviour rate + +# Print summary +tibble( + Parameter = c( + "Treatment schools", "Per school-level arm (CSE / CSE+Debiasing)", + "Surveyed students per school", + "Total students in chatbot randomisation", + "Per chatbot arm — Beta_2 (full sample)", + "Per chatbot arm — girls only (50% female)", + "Per 2×2 cell — Beta_3 (full sample)", + "Per 2×2 cell — girls only", + "Alpha (two-tailed)", "Target power", + "ICC — pregnancy", "ICC — overconfidence score", + "ICC — test scores", "ICC — depression/stress", + "ICC — knowledge/behaviour", + "Pregnancy base rate — 1 year", "Pregnancy base rate — 4 years", + "Overconfidence mean (pilot)", "Overconfidence SD (pilot)" + ), + Value = c( + n_schools_treatment, n_schools_per_arm, + n_students_per_school, + n_chatbot_total, + n_per_chatbot_arm, n_girls_per_arm, + n_per_cell_2x2, n_girls_per_cell, + alpha, power_target, + icc_pregnancy, icc_overconfidence, + icc_test_scores, icc_depression, icc_knowledge, + p0_preg_1yr, p0_preg_4yr, + mean_overconf, sd_overconf + ) +) |> + kable(caption = "Table 1: Key design and outcome parameters", align = c("l","r")) |> + kable_styling(bootstrap_options = c("striped","hover","condensed"), full_width = FALSE) |> + pack_rows("Study design", 1, 8) |> + pack_rows("Statistical conventions", 9, 10) |> + pack_rows("ICC estimates", 11, 15) |> + pack_rows("Outcome baselines", 16, 19) +``` + +--- + +# Analytical Framework: School Fixed Effects and Within-School Variance + +```{r within_school_variance} +# The residual variance driving precision is the *within-school* component: +# sigma^2_within = sigma^2_total × (1 – ICC) + + +within_school_sd <- function(sigma_total, icc) sigma_total * sqrt(1 - icc) + +tibble( + Outcome = c("Pregnancy — 1 year", "Pregnancy — 4 years", + "Overconfidence score", "Test scores", + "Depression / stress", "Knowledge / behaviour"), + sigma_total = c(sqrt(p0_preg_1yr * (1 - p0_preg_1yr)), + sqrt(p0_preg_4yr * (1 - p0_preg_4yr)), + sd_overconf, sd_test_scores, sd_depression, sd_knowledge), + icc = c(icc_pregnancy, icc_pregnancy, icc_overconfidence, + icc_test_scores, icc_depression, icc_knowledge) +) |> + mutate( + sigma_within = round(within_school_sd(sigma_total, icc), 4), + variance_kept_pct = round((1 - icc) * 100, 1), + variance_removed_pct = round(icc * 100, 1) + ) |> + rename( + `σ_total` = sigma_total, + `ICC` = icc, + `σ_within (with FE)` = sigma_within, + `Variance kept (%)` = variance_kept_pct, + `Variance removed (%)` = variance_removed_pct + ) |> + kable(digits = 4, + caption = "Table 2: Within-school variance after absorbing school fixed effects") |> + kable_styling(bootstrap_options = c("striped","hover","condensed"), full_width = FALSE) +``` + +# Analytical MDEs by Outcome + +```{r mde_function} +# Returns MDEs and Cohen's d (SE effects) for Estimates 1 and 2. +# For binary outcomes, pass sigma_total = sqrt(p0*(1-p0)). +# For standardized outcomes, pass sigma_total = 1 (SD). + +compute_mde <- function(sigma_total, icc, + n_arm, n_cell = n_arm / 2, + alpha = 0.05, power = 0.80) { + z_a <- qnorm(1 - alpha / 2) + z_b <- qnorm(power) + sig_w <- within_school_sd(sigma_total, icc) + + mde1 <- (z_a + z_b) * sig_w * sqrt(2 / n_arm) + mde2 <- (z_a + z_b) * sig_w * 2 / sqrt(n_cell) + + list( + sigma_within = sig_w, + mde_e1 = mde1, # MDE for Beta2 + mde_e2 = mde2, # MDE for Beta3 + d_e1 = mde1 / sig_w, # Cohen's d — Estimate 1 --> In Standard Deviations + d_e2 = mde2 / sig_w # Cohen's d — Estimate 2 --> In Standard Deviations + ) +} +``` + +```{r compute_all_outcomes} +# Apply to all outcomes + +# Pregnancy (binary) +pg1 <- compute_mde(sqrt(p0_preg_1yr * (1 - p0_preg_1yr)), icc_pregnancy, + n_per_chatbot_arm, n_per_cell_2x2) +pg4 <- compute_mde(sqrt(p0_preg_4yr * (1 - p0_preg_4yr)), icc_pregnancy, + n_per_chatbot_arm, n_per_cell_2x2) + +# Overconfidence score (continuous, scale points) +oc <- compute_mde(sd_overconf, icc_overconfidence, + n_per_chatbot_arm, n_per_cell_2x2) + +# Test scores — full sample and girls only +ts_full <- compute_mde(sd_test_scores, icc_test_scores, + n_per_chatbot_arm, n_per_cell_2x2) +ts_girls <- compute_mde(sd_test_scores, icc_test_scores, + n_girls_per_arm, n_girls_per_cell) + +# Depression / stress — full sample and girls only +dep_full <- compute_mde(sd_depression, icc_depression, + n_per_chatbot_arm, n_per_cell_2x2) +dep_girls <- compute_mde(sd_depression, icc_depression, + n_girls_per_arm, n_girls_per_cell) + +# Knowledge / behaviour index — full sample and girls only +kb_full <- compute_mde(sd_knowledge, icc_knowledge, + n_per_chatbot_arm, n_per_cell_2x2) +kb_girls <- compute_mde(sd_knowledge, icc_knowledge, + n_girls_per_arm, n_girls_per_cell) +``` + +--- + +# Summary Table: MDEs for Estimands 1 and 2 {.tabset} + +The table below is the primary output of this document. Each row is an outcome; the two columns report the MDE for the **main chatbot effect** ($\beta_2$) and for the **chatbot × debiasing interaction** ($\beta_3$). + +```{r summary_mde_table} +# Claude helped me with this table format for a tidy export +outcomes_list <- list(pg1, pg4, oc, + ts_full, ts_girls, + dep_full, dep_girls, + kb_full, kb_girls) + +outcome_names <- c( + "Pregnancy rate — 1 year", + "Pregnancy rate — 4 years", + "Overconfidence score", + "Test scores", + "Test scores — girls only", + "Depression / stress", + "Depression / stress — girls only", + "Knowledge / behaviour index", + "Knowledge / behaviour — girls only" +) + +# Natural unit labels for each outcome +units <- c("pp", "pp", "pts", "SD", "SD", "SD", "SD", "pp", "pp") + +# ICC used for each outcome (for display) +icc_used <- c(icc_pregnancy, icc_pregnancy, icc_overconfidence, + icc_test_scores, icc_test_scores, + icc_depression, icc_depression, + icc_knowledge, icc_knowledge) + +# N per arm used for Estimand 1 (halved for girls-only rows) +n_arm_used <- c(n_per_chatbot_arm, n_per_chatbot_arm, n_per_chatbot_arm, + n_per_chatbot_arm, n_girls_per_arm, + n_per_chatbot_arm, n_girls_per_arm, + n_per_chatbot_arm, n_girls_per_arm) + +# Helper: format MDE with units and Cohen's d +fmt_mde <- function(r, units_label) { + sprintf("%.4f %s\n(d = %.4f)", r$mde_e1, units_label, r$d_e1) +} +fmt_mde2 <- function(r, units_label) { + sprintf("%.4f %s\n(d = %.4f)", r$mde_e2, units_label, r$d_e2) +} + +mde_tbl <- tibble( + Outcome = outcome_names, + `ICC (ρ)` = icc_used, + `N per arm (β₂)` = scales::comma(n_arm_used), + `σ_within` = round(sapply(outcomes_list, `[[`, "sigma_within"), 4), + `MDE — β₂ (Chatbot only)` = mapply(fmt_mde, outcomes_list, units), + `MDE — β₃ (Chatbot × Debiasing)` = mapply(fmt_mde2, outcomes_list, units) +) + +mde_tbl |> + kable( + caption = paste0( + "Table 3: Minimum Detectable Effects — Estimates 1 and 2 ", + "(80% power, α = 0.05, two-tailed)" + ), + align = c("l","c","r","r","l","l") + ) |> + kable_styling(bootstrap_options = c("striped","hover","condensed"), + full_width = TRUE) |> + pack_rows("Binary outcomes", 1, 2) |> + pack_rows("Continuous outcomes", 3, 9) |> + column_spec(5, width = "18em") |> + column_spec(6, width = "18em") |> + footnote( + general = paste0( + "MDE(β₂) = (z_β + z_{α/2}) × σ_within × √(2/N_arm). ", + "MDE(β₃) = (z_β + z_{α/2}) × σ_within × 2/√(n_cell). ", + "σ_within = σ_total × √(1 – ICC). ", + "MDE(β₃)/MDE(β₂) = 2 by construction for balanced 2×2. ", + "Girls-only rows assume 50% female (N halved). ", + "pp = percentage points; pts = scale points; SD = standard deviation units." + ), + general_title = "Notes: " + ) +``` + + + +# Open Design Variation: Saturation Design (25%, 50%, 75%) + +```{r saturation_analytical} +# Analytical information loss comparison + +# Uniform 50/50: all 450 schools at s = 0.50 +info_uniform <- n_schools_treatment * n_students_per_school * 0.50 * 0.50 + +# Three-level saturation: equal thirds (150 schools each) at 25%, 50%, 75% +sat_levels <- c(0.25, 0.50, 0.75) +schools_per_level <- n_schools_treatment / length(sat_levels) # 150 schools each + +info_saturation <- sum( + schools_per_level * n_students_per_school * sat_levels * (1 - sat_levels) +) + +info_ratio <- info_saturation / info_uniform # relative efficiency (< 1) +se_inflation <- 1 / sqrt(info_ratio) # SE multiplier (> 1); same for all outcomes + +info_loss_pct <- (1 - info_ratio) * 100 # % information lost + +# Under saturation, MDE scales up by se_inflation for every outcome + +# Per-outcome MDE table + +sat_tbl <- tibble( + Outcome = outcome_names, + + # Columns 1–2: design-level quantities — identical for every outcome + `Info loss` = sprintf("−%.1f%%", info_loss_pct), + `SE inflation` = sprintf("×%.4f", se_inflation), + + # Columns 3–4: MDE for β₂ (main chatbot effect) under each design + `MDE β₂ — uniform` = mapply(function(r, u) + sprintf("%.4f %s", r$mde_e1, u), + outcomes_list, units), + + `MDE β₂ — saturation` = mapply(function(r, u) + sprintf("%.4f %s\n(+%.1f%%)", r$mde_e1 * se_inflation, u, + (se_inflation - 1) * 100), + outcomes_list, units), + + # Columns 5–6: MDE for β₃ (chatbot × debiasing interaction) under each design + `MDE β₃ — uniform` = mapply(function(r, u) + sprintf("%.4f %s", r$mde_e2, u), + outcomes_list, units), + + `MDE β₃ — saturation` = mapply(function(r, u) + sprintf("%.4f %s\n(+%.1f%%)", r$mde_e2 * se_inflation, u, + (se_inflation - 1) * 100), + outcomes_list, units) +) + +sat_tbl |> + kable( + caption = paste0( + "Table 5: MDE Comparison — Uniform 50/50 vs. Saturation 25%/50%/75% Design ", + "(80% power, α = 0.05, two-tailed)" + ), + align = c("l", "c", "c", "r", "r", "r", "r") + ) |> + kable_styling(bootstrap_options = c("striped", "hover", "condensed"), + full_width = TRUE) |> + add_header_above(c( + " " = 1, + "Design cost (constant across outcomes)" = 2, + "Estimate 1: Main chatbot effect (β₂)" = 2, + "Estimate 2: Chatbot × Debiasing (β₃)" = 2 + )) |> + pack_rows("Binary outcomes", 1, 2) |> + pack_rows("Continuous outcomes", 3, 9) |> + footnote( + general = paste0( + "Information loss and SE inflation are design-level quantities — ", + "identical for all outcomes since the saturation assignment is independent of the outcome. ", + "MDE(saturation) = MDE(uniform) × SE inflation. ", + "MDE(β₃) = 2 × MDE(β₂) by construction for a balanced 2×2 design. ", + "% increase in parentheses = (se_inflation − 1) × 100 = +", + round((se_inflation - 1) * 100, 1), "% for all cells. ", + "pp = percentage points; pts = scale points; SD = standard deviation units." + ), + general_title = "Notes: " + ) +``` + +# Power Simulations + +```{r powersimu_function} +# R translation of the Stata powersimu program + +powersimu <- function(n_clusters, cluster_size, b_spill, rho) { + + # 1. Create clusters: cluster-level error drives ICC + u_cluster <- rnorm(n_clusters, mean = 0, sd = sqrt(rho)) + + # 2. Assign saturation levels randomly from {0, 0.25, 0.50, 0.75} + saturation <- sample(c(0, 0.25, 0.50, 0.75), n_clusters, replace = TRUE) + + # 3. Expand to individuals + id_j <- rep(seq_len(n_clusters), each = cluster_size) + u_j <- rep(u_cluster, each = cluster_size) + sat_j <- rep(saturation, each = cluster_size) + + # 4. Individual-level error: e_ij ~ N(0, sqrt(1 - rho)) + e_ij <- rnorm(n_clusters * cluster_size, mean = 0, sd = sqrt(1 - rho)) + + # 5. Assign individual treatment based on cluster saturation + treated <- rbinom(n_clusters * cluster_size, size = 1, prob = sat_j) + + # 6. Generate outcome + y <- 0.5 * treated + b_spill * sat_j + u_j + e_ij + df <- data.frame(id_j = factor(id_j), treated, sat_j, y) + + # 7. Cluster-robust regression + fit <- lm(y ~ treated + sat_j, data = df) + vcov_cl <- sandwich::vcovCL(fit, cluster = ~ id_j) + coef_tbl <- lmtest::coeftest(fit, vcov = vcov_cl) + + # P-value for saturation coefficient + p_val <- coef_tbl["sat_j", "Pr(>|t|)"] + + # Rejection indicator + as.integer(p_val < 0.05) +} +``` + + +```{r study_sim_grid, cache=TRUE} +# Study-specific simulation: 450 schools × 90 students + +set.seed(28052026) +n_reps_study <- 1000 + +b_spill_grid <- c(0.10, 0.20, 0.30, 0.40, 0.50) + +# Named vector: rho value (one per study outcome group) +rho_scenarios <- c( + "Pregnancy (ρ = 0.05)" = icc_pregnancy, + "Overconfidence (ρ = 0.065)" = icc_overconfidence, + "Depression (ρ = 0.10)" = icc_depression, + "Test scores (ρ = 0.20)" = icc_test_scores +) + +# Run grid: for each ICC scenario × each b_spill value, run n_reps_study simulations +sim_grid <- map_dfr(names(rho_scenarios), function(scenario) { + rho_val <- rho_scenarios[[scenario]] + + power_vec <- map_dbl(b_spill_grid, function(bs) { + res <- lapply(1:n_reps_study, function(x) + powersimu( + n_clusters = n_schools_treatment, # 450 + cluster_size = n_students_per_school, # 90 + b_spill = bs, + rho = rho_val + ) + ) + mean(unlist(res)) + }) + + tibble(Scenario = scenario, b_spill = b_spill_grid, power = power_vec) +}) +``` + +## Sim Results + +```{r sim_results_table} +# Add 80% power reference row via row_spec highlighting. + +sim_wide <- sim_grid |> + mutate(power_fmt = sprintf("%.1f%%", power * 100)) |> + select(b_spill, Scenario, power_fmt) |> + pivot_wider(names_from = Scenario, values_from = power_fmt) |> + rename(`b_spill (SD)` = b_spill) + +sim_wide |> + kable( + caption = paste0( + "Table 6: Simulated Power to Detect the Spillover Effect — ", + "Saturation Design (25% / 50% / 75%), ", + n_reps_study, " replications per cell" + ), + align = "c" + ) |> + kable_styling(bootstrap_options = c("striped", "hover", "condensed"), + full_width = FALSE) |> + add_header_above(c(" " = 1, "Simulated power by ICC scenario" = 4)) |> + footnote( + general = paste0( + "Power = fraction of ", n_reps_study, " simulations rejecting H₀: b_spill = 0 ", + "at α = 0.05 (two-tailed, cluster-robust SEs via sandwich::vcovCL). ", + "N = ", scales::comma(n_schools_treatment * n_students_per_school), " students ", + "across ", n_schools_treatment, " schools. ", + "Saturation levels drawn uniformly from {0, 0.25, 0.50, 0.75} per school. ", + "Direct treatment effect fixed at 0.5 SD. ", + "b_spill is in normalised units where σ_total = 1." + ), + general_title = "Notes: " + ) +``` \ No newline at end of file diff --git a/cse-dr/powercalc_NG_approach.md b/cse-dr/powercalc_NG_approach.md new file mode 100644 index 0000000..787814b --- /dev/null +++ b/cse-dr/powercalc_NG_approach.md @@ -0,0 +1,181 @@ +# CSE AI Chatbot Power Analysis + +**Author:** Nandita (edited using Claude) · **First review by:** Claude +ESI en Valores RCT (Dominican Republic) · chatbot component + + +> Note: this document lays out the proposed approach for review. Results are held until the approach is signed off, then we run and fill them in. The *Claude comment* blocks under each point are a first review. **Luiza:** please mark the response line under each point — change `[ ]` to `[x]` for Yes / Discuss / No and add comments. Your edits are the record. + +--- + +## What's available (inputs) + +**Design (from the study design doc):** + +| Parameter | Value | +|---|---| +| Treatment schools | 450 (225 Regular CSE / 225 CSE+Debiasing) | +| Surveyed students per school | 90 (30 per grade × 3 grades) | +| Total students in chatbot randomization | 40,500 | +| Per chatbot arm (β₂) | 20,250 | +| Per 2×2 cell (β₃) | 10,125 | +| Chatbot randomization | Individual student, within school, stratified by CSE arm | +| Power / significance | 80% / 5%, two-sided | +| ICC — pregnancy | 0.050 | +| ICC — overconfidence score | 0.065 | +| ICC — depression / knowledge | 0.100 | +| ICC — test scores | 0.200 | +| Pregnancy base rate (1 yr / 4 yr) | 0.056 / 0.168 | +| Overconfidence mean / SD | 10.4 / 27.7 | + +**Other documents for reference:** +- Existing spreadsheet — MDEs for the school-level estimands (CSE vs control; debiasing vs CSE). +- Tomás's `PowerCalcsChatbot.Rmd` — analytic MDEs (`compute_mde`, lines 185–204), the saturation information-loss block (lines 329–342), and a power simulation (`powersimu`, lines 413–448). + +--- + +## Assumptions + +| Assumption | Choice | Type / note | +|---|---|---| +| Spillover channel | Untreated students benefit linearly in their school's treated share | Modelling choice different from the note | +| Spillover magnitude | Swept 0.05–0.30 SD | **Placeholder** — replace with another estimate when available | +| Saturation levels | 25% / 50% / 75%, equal thirds, stratified by CSE arm | Design option under consideration | +| Debiasing coding | Centered (`Deb − 0.5`) | Spec choice so β₂ is the pooled effect (see §3) | +| Estimator | School fixed effects + cluster-robust SEs | Standard library| +| Female share (girls-only rows) | 50% | From design doc and existing calc | +| Simulation replications | 1,000 | From existing calc | + + +> **Luiza — do these assumptions look right?** `[ ] Yes` · `[ ] Discuss` · `[ ] No` — comments: + +--- + +## 1. The estimands we target + +The chatbot is randomized at the individual student level *within* the 450 treatment schools, on top of the school-level CSE / CSE+debiasing split. That gives three quantities of interest: + +- **β₂ — direct effect of chatbot access**, averaged across both CSE arms (the study's Estimand 1: cells B+D vs A+C). +- **β₃ — chatbot × debiasing interaction** (Estimand 2): is the chatbot more effective alongside debiasing? +- **δ — spillover / saturation effect**: does a student's outcome move with the *share* of their peers who have the chatbot? From the background note, my understanding is that the whole reason we're using saturation levels is to measure spillovers. + +> **Claude comment:** Treating δ as first-class is the right call. Two caveats. (1) The design doc also raises *nonlinear* network/equilibrium effects (norms shifting only past some saturation threshold); a single linear δ can't capture those, so it's worth running at least one nonlinear saturation form as a robustness check. (2) With ~9 outcome×subgroup rows and three estimands each, multiple-testing will matter for interpretation even though it doesn't change MDEs — pre-specifying primary outcomes now would help. + +> **Luiza — agree?** `[ ] Yes` · `[ ] Discuss` · `[ ] No` — comments: + +--- + +## 2. The model + +We simulate the full design and generate each student's outcome from an explicit model. Let `T` = chatbot access (0/1), `Deb` = debiasing school (0/1), and `s_g` = realized treated share in school *g*: + +``` +Y_ig = γ_g + β2·T_ig + β3·(Deb_g − 0.5)·T_ig + δ·s_g·(1 − T_ig) + ε_ig +``` + +- `γ_g` is a school-level term carrying the between-school variance; its size is set by each outcome's **ICC** (variance ICC), and `ε` carries the within-school variance (1 − ICC). So the total variance is 1 and effects read in SD units. +- `δ·s_g·(1 − T_ig)` is the **spillover**: untreated students benefit in proportion to how saturated their school is (information passed from treated peers). This single term is what lets us study the bias and the spillover estimand. +- The debiasing indicator is **centered** (`Deb − 0.5`) so that β₂ is the *average* chatbot effect across both CSE arms (see §3). + +Saturation is set per school: the uniform design fixes `s_g = 0.5` everywhere; the saturation design assigns `s_g ∈ {0.25, 0.50, 0.75}` in equal thirds, stratified by CSE arm. + +> **Claude comment:** The spillover here accrues only to *untreated* students (free-riding). That's a reasonable lead case, but it's what drives the headline "−0.5·δ bias," so flag it as a functional-form assumption and stress-test it against a *symmetric* spillover (benefiting treated and untreated alike), under which the FE estimate is **not** biased — the two cases give very different recommendations. Also: real spillover probably runs at the classroom/grade level, not the whole school; if so, the relevant clustering and the saturation variation change, and school-level modelling may overstate the between-cluster variation we have. Minor: adding the spillover term mechanically raises total outcome variance, so hold σ_total fixed when reading MDEs in "SD units." + +> **Luiza — agree?** `[ ] Yes` · `[ ] Discuss` · `[ ] No` — comments: + +--- + +## 3. Estimation — two specifications + +Because the chatbot varies within school, the natural estimator absorbs **school fixed effects**, which removes the between-school variance for free. We run two specifications (with school-clustered SEs): + +**(A) Direct-effect spec (school FE):** +``` +Y ~ chatbot + (Deb_c · chatbot) | school +``` +recovers β₂ and β₃. *Centering: With a raw 0/1 debiasing variable, the chatbot coefficient would be the effect in regular-CSE schools only, estimated off half the schools with an SE root2 larger. Centering makes it the pooled average. + +**(B) Spillover spec (no FE; saturation as a regressor):** +``` +Y ~ chatbot + school_share + (Deb_c · chatbot) [clustered by school] +``` +The coefficient on `chatbot` recovers the **unbiased** direct effect; the coefficient on `school_share` recovers the spillover δ. This is only identified when `s_g` varies — i.e. under the saturation design. + +> **Claude comment:** Centering is correct and standard — low risk. The thing to watch is Spec B: dropping school FE to identify δ puts the between-school variance back into the error, so it's less efficient for β₂ and leans entirely on saturation being randomly assigned (it is, so no bias — but report both specs and the gap between them, since that gap *is* the spillover story). If spillover can also reach treated students, Spec B as written (`school_share` main effect only) won't fully separate direct from spillover effects — we'd want the `school_share × chatbot` term too. And for rare binary outcomes (pregnancy at 5.6%), FE on a linear-probability model is an approximation: fine for MDEs, but the final analysis may want a GLM or a robustness check. + +> **Luiza — agree?** `[ ] Yes` · `[ ] Discuss` · `[ ] No` — comments: + +--- + +## 4. Simulation + Validation + +We build a virtual study with a known planted effect, run the estimator, and record whether it's detected; the fraction detected over many repetitions will be the power we need. + +### How we will build the virtual study: + +Each "study" is a table of 40,500 students that we construct from scratch — and because we plant the true effects ourselves, we know the right answer to check against. + +0. **Set the knobs we control:** the true effects (β₂, β₃, δ), the outcome's ICC, and the design (uniform 50/50 or saturation 25/50/75). We choose these, so we know them. +1. **Make the schools and assign debiasing:** create 450 schools; mark 225 at random as debiasing, 225 as regular CSE (a school-level decision). +2. **Give each school its chatbot share:** uniform -> every school 50%; saturation -> 25% / 50% / 75% to equal thirds of schools (150 each), done separately within each CSE arm so they stay balanced. +3. **Pick which students get the chatbot, within each school:** for a 75%-share school we treat exactly 68 of its 90 students (90 × 0.75 ≈ 68), chosen at random — exact count, no accidental imbalance. +4. **Create the noise in two layers** (this is where the ICC enters): one *school-level* draw shared by all 90 students (variance = ICC, which makes classmates resemble each other) plus one *individual* draw per student (variance = 1 − ICC). They sum to total variance 1, so the outcome is in SD units. For ICC = 0.10 that's a school SD of √0.10 ≈ 0.32 and an individual SD of √0.90 ≈ 0.95. +5. **Build each student's outcome** by plugging into the model equation (§2): school draw + chatbot effect (if treated) + interaction (in debiasing schools) + spillover (for untreated, ∝ school share) + individual noise. +6. **Result:** one complete dataset (one row per student) ready to hand to the regression. + +*Worked example by claude based on approach shared* (β₂ = 0.05, δ = 0.10, ICC = 0.10, a debiasing school assigned 50% saturation that drew a school value of +0.20): +- A **treated** student with individual noise −0.30 → Y = 0.20 + 0.05 − 0.30 = **−0.05**. +- An **untreated** classmate with individual noise +0.10 → Y = 0.20 + (0.10 × 0.50) + 0.10 = **0.35** (the +0.05 is the spillover from sitting in a half-saturated school). + +We then repeat steps 1 to 6 a thousand times with fresh random draws; the share of runs in which the estimator detects the effect is the power. + +For validation, we check if: (i) simulated standard errors match the analytical formula to three decimals; (ii) the interaction SE comes out exactly 2× the main-effect SE, as theory predicts; (iii) the false-positive rate is 0.05 and every planted parameter is recovered. + +> **Claude comment:** Validation is solid for the simple case — but note it currently checks the *no-spillover* analytic formula and doesn't independently verify the spillover-spec SEs, so I'd add a second benchmark for δ (e.g. a school-level aggregated regression) before trusting those power numbers. Errors are assumed Gaussian; for rare binary outcomes the finite-sample coverage should be checked separately. + +> **Luiza — agree?** `[ ] Yes` · `[ ] Discuss` · `[ ] No` — comments: + +--- + +## NG analysis on how it compares to Tomás's approach and can build on it + +**What we keep:** his insight that, because the chatbot is randomized within school, the relevant noise is the within-school SD `σ_within = σ_total·√(1 − ICC)`. We reproduce his analytic MDEs (`compute_mde`, lines 185–204). + +**What we add, point by point:** + +1. **An explicit spillover term.** His model (and the analytic MDEs) have no peer channel, so the bias that motivates the saturation design is invisible. Adding `δ·s_g·(1−T)` lets us quantify the `≈ 0.5·δ` attenuation and recover an unbiased direct effect. + + > **Claude comment:** This is the core contribution and I think it's right — just keep the functional-form caveat from §2 attached to it, so it doesn't read as the only possible spillover model. + + > **Luiza — agree?** `[ ] Yes` · `[ ] Discuss` · `[ ] No` — comments: + +2. **Power for the spillover itself.** His `powersimu` (lines 413–448) estimates a spillover coefficient, but under different assumptions from the analytic block. We power δ within the same framework. + + > **Claude comment:** This is probably the most decision-relevant finding. If δ is underpowered for high-ICC outcomes like test scores, that's an argument for *more* or *wider* saturation levels (e.g. 0/33/67/100) — worth simulating that option rather than just reporting low power. + + > **Luiza — agree?** `[ ] Yes` · `[ ] Discuss` · `[ ] No` — comments: + +3. **An explicit, matched specification.** `mde1` (line 192) is implicitly the pooled effect; we make that hold by centering debiasing, so the regression the team runs matches the power calc. + + > **Claude comment:** Clearly correct and low-risk. The only thing to confirm is that the *actual* analysis code uses the centered coding — otherwise the MDE and the estimate won't line up, which is exactly the gap this point is meant to close. + + > **Luiza — agree?** `[ ] Yes` · `[ ] Discuss` · `[ ] No` — comments: + +4. **One simulation for everything.** His analytic loss (lines 329–342) and his simulation (lines 413–448) use different assumptions (direct effect fixed at 0.5 SD, line 433; `lm` without school FE, line 437). We produce the MDEs, saturation loss, bias, and spillover power from one consistent engine, validated against his formulas. + + > **Claude comment:** Right principle; the cost is speed and transparency (one big engine is harder to eyeball than a one-line formula). Mitigate by validating the engine against Tomás's closed-form numbers at each step, so consolidation can't quietly hide a bug. + + > **Luiza — agree?** `[ ] Yes` · `[ ] Discuss` · `[ ] No` — comments: + + +> **Claude comment (overall):** The approach is sound and a genuine improvement on a correct foundation. The single biggest risk is the spillover *functional form* (free-riding, linear, school-level): the bias and recovery conclusions are conditional on it, so I'd make testing 2–3 alternative spillover structures part of v1 rather than a follow-up. Second priority is settling whether spillover/saturation lives at the classroom or school level, since that changes what variation we actually have. + +--- + +## Overall: green light to proceed? + +If the above sounds reasonable, Nandita takes a first pass at the code on top of Tomás's framework and shares it for Luiza and Tomás to check. + +> **Luiza — overall decision:** `[ ] Yes, go ahead` · `[ ] Let's discuss first` · `[ ] No` — comments: + +*— Nandita (edited using Claude)* diff --git a/cse-dr/powercalc_NG_approach/baird_power.R b/cse-dr/powercalc_NG_approach/baird_power.R new file mode 100644 index 0000000..a47f7c2 --- /dev/null +++ b/cse-dr/powercalc_NG_approach/baird_power.R @@ -0,0 +1,92 @@ +# Baird, Bohren, McIntosh & Ozler (2016) randomized-saturation power functions. +# Source: PDEL "R Implementation.R" (base R, no packages). +# https://pdel.ucsd.edu/_files/R%20Implementation.R +# One bug fixed below: in power_ind, `zeros(1,length(pi))` -> `rep(0,length(pi))`. +# +# Functions give analytic MDEs (Theorems 1-3): +# power_pooled : pooled treatment (MDE_T), pooled spillover (MDE_S), +# treatment-only-with-pure-control (MDE_Tonly) +# power_slope : slope MDEs (how spillover changes with saturation): MDSE_T / MDSE_S +# power_ind : individual-saturation treatment / spillover MDEs +# Args: n=cluster size, C=#clusters, alpha, gamma=power, tau=between-cluster var, +# sigma=within-cluster var, pi=saturations, f=fraction of clusters at each. +# +# NOTE: written for your R/RStudio (this was not executed in-session; the +# numbers in the comments were verified via the identical Python implementation). + +power_pooled = function(n,C,alpha,gamma,tau,sigma,pi,f){ + t_alpha = qt(1 - alpha/2,n*C-3); t_gamma = qt(gamma,n*C-3) + mu_ind = rep(0,length(pi)); eta_ind = rep(0,length(pi)) + for(i in 1:length(pi)){ mu_ind[i]=pi[i]*f[i]; eta_ind[i]=pi[i]^2*f[i] } + mu = sum(mu_ind); eta = sum(eta_ind) + psi = 0; if(pi[1]==0) psi = f[1] + muS = 1 - mu - psi + etaT = (eta-mu^2)/(1-psi)-(psi/(1-psi)^2)*mu^2 + varN = tau+sigma; varCo = (n-1)*tau + Var = 1/(n*C)*(varCo*(1/(psi*(1-psi))+(1-psi)/(mu ^2)*etaT)+varN*(psi+mu )/(mu *psi)) + VarS = 1/(n*C)*(varCo*(1/(psi*(1-psi))+(1-psi)/(muS^2)*etaT)+varN*(psi+muS)/(muS*psi)) + MDE_T = (t_alpha+t_gamma)*Var^0.5 + MDE_S = (t_alpha+t_gamma)*VarS^0.5 + Var_T = 1/(n*C)*(varCo*(eta-mu^2)/(mu^2*(1-mu)^2)+varN/(mu*(1-mu))) + MDE_Tonly = (t_alpha+t_gamma)*Var_T^0.5 + return(list(MDE_T=MDE_T, MDE_S=MDE_S, MDE_Tonly=MDE_Tonly)) +} + +power_slope = function(n,C,alpha,gamma,tau,sigma,pi,f,j,k){ + t_alpha = qt(1 - alpha/2,n*C-3); t_gamma = qt(gamma,n*C-3) + mu_ind = rep(0,length(pi)); p_ind = rep(0,length(pi)) + for(i in 1:length(pi)){ mu_ind[i]=pi[i]*f[i]; p_ind[i]=(1-pi[i])*f[i] } + varN = tau+sigma; varCo = (n-1)*tau + Var_T = (varCo*(1/f[j]+1/f[k])+varN*(1/mu_ind[j]+1/mu_ind[k]))/(n*C) + Var_S = (varCo*(1/f[j]+1/f[k])+varN*(1/p_ind[j] +1/p_ind[k]))/(n*C) + MDSE_T = ((t_alpha+t_gamma)/(pi[k]-pi[j]))*Var_T^0.5 + MDSE_S = ((t_alpha+t_gamma)/(pi[k]-pi[j]))*Var_S^0.5 + return(list(MDSE_T=MDSE_T, MDSE_S=MDSE_S)) +} + +power_ind = function(n,C,alpha,gamma,tau,sigma,pi,f,p){ + t_alpha = qt(1 - alpha/2,n*C-3); t_gamma = qt(gamma,n*C-3) + mu_ind = rep(0,length(pi)); p_ind = rep(0,length(pi)) # fixed: was zeros(1,length(pi)) + for(i in 1:length(pi)){ mu_ind[i]=pi[i]*f[i]; p_ind[i]=(1-pi[i])*f[i] } + psi = 0; if(pi[1]==0) psi = f[1] + varN = tau+sigma; varCo = (n-1)*tau + MDE_ind_T = (t_alpha+t_gamma)*(1/(n*C)*(varCo*(1/f[p]+1/psi)+varN*(1/mu_ind[p]+1/psi)))^0.5 + MDE_ind_S = (t_alpha+t_gamma)*(1/(n*C)*(varCo*(1/f[p]+1/psi)+varN*(1/p_ind[p] +1/psi)))^0.5 + return(list(MDE_ind_T=MDE_ind_T, MDE_ind_S=MDE_ind_S)) +} + +# ============================================================================ +# Wiring to our study (ESI en Valores chatbot): spillover-SLOPE MDE per outcome, +# comparing equal-thirds {25/50/75} vs extreme-weighted {20/50/80} @ 40/20/40. +# (The slope is the relevant spillover estimand when there is no pure-control +# saturation; it is identified from variation across saturations.) +# ============================================================================ +if (sys.nframe() == 0) { # run only when sourced/executed directly + + # (a) paper-replication check (Baird et al. Table 2, col 5): + r <- power_pooled(10,100,.05,.8,.1,.9, c(0,.25,.5,.75,1), rep(.2,5)) + s <- power_slope (10,100,.05,.8,.1,.9, c(0,.25,.5,.75,1), rep(.2,5), 2, 4) + cat(sprintf("paper check MDE_T=%.4f MDE_S=%.4f slope MDSE_T=%.4f\n", + r$MDE_T, r$MDE_S, s$MDSE_T)) + # expected (from validated Python run): MDE_T=0.3179 MDE_S=0.3387 slope=1.0592 + + # (b) our outcomes + n <- 90; C <- 450 + outcomes <- list(c("Pregnancy", 0.050), + c("Overconfidence", 0.065), + c("Depression/Knowledge",0.100), + c("Test scores", 0.200)) + cat(sprintf("\nSpillover-slope MDE (SD), 80%% power, a=0.05, %d schools x %d students:\n", C, n)) + cat(sprintf("%-22s %8s %14s %16s\n","Outcome","ICC","equal thirds","extreme 40/20/40")) + for (o in outcomes) { + name <- o[1]; icc <- as.numeric(o[2]); tau <- icc; sig <- 1 - icc + eq <- power_slope(n,C,.05,.8,tau,sig, c(.25,.50,.75), c(1/3,1/3,1/3), 1, 3) + ex <- power_slope(n,C,.05,.8,tau,sig, c(.20,.50,.80), c(.40,.20,.40), 1, 3) + cat(sprintf("%-22s %6.3f %12.4f %15.4f\n", name, icc, eq$MDSE_S, ex$MDSE_S)) + } + # expected (validated via Python): + # Pregnancy 0.050 0.1819 0.1428 + # Overconfidence 0.065 0.1983 0.1549 + # Depression/Knowledge 0.100 0.2320 0.1799 + # Test scores 0.200 0.3085 0.2373 +} diff --git a/cse-dr/powercalc_NG_approach/spillover_power_check.Rmd b/cse-dr/powercalc_NG_approach/spillover_power_check.Rmd new file mode 100644 index 0000000..a85c1ab --- /dev/null +++ b/cse-dr/powercalc_NG_approach/spillover_power_check.Rmd @@ -0,0 +1,563 @@ +--- +title: "Spillover Power & the Saturation-Design" +subtitle: "AI Chatbot Component ESI en Valores RCT" +author: "Nandita" +date: "`r format(Sys.Date())`" +output: + html_document: + toc: true + toc_float: true + number_sections: true + code_folding: show +--- + +```{r setup, include=FALSE} +knitr::opts_chunk$set(echo = TRUE, message = FALSE, warning = FALSE) +``` + +# Purpose of this document + +1. Shows, with two independent tools, **how much power we have to detect a spillover** under different saturation schemes: + - the **Baird et al. analytic MDE formulas** (their published R code), and + - the **`RCT2`** package (Jiang, Imai & Malani 2023) + +------------------------------------------------------------------------ + +# Inputs, notation, and assumptions + +```{r tbl-inputs} +library(knitr); library(kableExtra) +inputs <- data.frame( + Parameter = c("Treatment schools (C)", "Students per school (n)", "Total students", + "Significance (alpha)", "Power (gamma)", + "ICC — pregnancy", "ICC — overconfidence", + "ICC — depression / knowledge", "ICC — test scores", + "Pregnancy base rate (1 yr / 4 yr)", "Overconfidence mean / SD", + "Female share (girls-only rows)"), + Value = c("450", "90", "40,500", "0.05 (two-sided)", "0.80", + "0.050", "0.065", "0.100", "0.200", + "0.056 / 0.168", "10.4 / 27.7", "0.50"), + Source = c("design doc", "design doc (30/grade x 3 grades)", "design doc", + "convention", "convention", + "ENHOGAR (conservative)", "2024 pilot (N = 800)", + "health literature / pilot", "Pruebas Nacionales", + "WDI / Dupas 2018", "2024 pilot", "design doc"), + check.names = FALSE) +kable(inputs, caption = "Table 1. Design inputs and data (with sources)") |> + kable_styling(bootstrap_options = c("striped","hover","condensed"), full_width = FALSE) +``` + +```{r tbl-assumptions} +assum <- data.frame( + Assumption = c("Spillover channel", "Spillover sizes swept", "Saturation designs", + "Debiasing coding", "Estimator (chatbot effect)", + "Estimator (spillover)", "Simulation reps"), + Choice = c("untreated outcome rises by spillover x (school's treated share)", + "0, 0.10, 0.20, 0.30 SD (0 = no spillover)", + "uniform 50/50; equal thirds 25/50/75; extreme 20/50/80 @ 40/20/40", + "centered (Deb - 0.5) so the chatbot coefficient is the pooled effect", + "school fixed effects + cluster-robust SEs", + "controls for the school share; no school FE (strata FE + cluster SEs)", + "300 per cell (raise for smoother power)"), + Type = c("our modelling choice (Baird-style)", "placeholder — replace with pilot", + "design options compared", "specification choice", + "standard", "standard", "numerical"), + check.names = FALSE) +kable(assum, caption = "Table 2. Modelling assumptions (our choices)") |> + kable_styling(bootstrap_options = c("striped","hover","condensed"), full_width = FALSE) +``` + +```{r tbl-notation} +notation <- data.frame( + Symbol = c("n", "C", "tau", "sigma", "varN", "varCo", "pi", "f", + "mu_ind", "p_ind", "j, k", "t_alpha", "t_gamma", "Var_S", "MDSE_S"), + Meaning = c("students per school", "number of schools", + "between-school variance", "within-school variance", + "total individual variance", "clustering penalty", + "saturation levels (treated share per bin)", + "share of schools at each saturation", + "expected treated share in a bin", "expected untreated share in a bin", + "the two saturation bins compared for the slope", + "test critical value", "power critical value", + "sampling variance of the spillover-slope estimate", + "minimum detectable spillover slope"), + `Formula / value` = c("90", "450", "= ICC", "= 1 - ICC", "tau + sigma (= 1)", + "(n - 1) * tau", "e.g. c(.25,.50,.75)", "e.g. c(1/3,1/3,1/3)", + "pi * f", "(1 - pi) * f", "the extreme bins (1 and 3)", + "qt(1 - alpha/2, nC - 3)", "qt(gamma, nC - 3)", + "[varCo*(1/f_j+1/f_k) + varN*(1/p_j+1/p_k)] / (nC)", + "(t_alpha + t_gamma) * sqrt(Var_S) / (pi_k - pi_j)"), + Source = c("design", "design", "definition (= ICC)", "definition", + "Baird et al.", "Baird et al.", "design choice", "design choice", + "Baird et al.", "Baird et al.", "our choice", + "Baird et al.", "Baird et al.", "Baird et al.", "Baird et al."), + check.names = FALSE) +kable(notation, caption = "Table 3. Notation used in the formulae") |> + kable_styling(bootstrap_options = c("striped","hover","condensed"), full_width = FALSE) +``` + +------------------------------------------------------------------------ + +# Background: the design and why spillovers matter + +The chatbot is randomized in two stages: + +- **Stage 1 (school level):** schools are assigned to *Regular CSE* vs *CSE + Debiasing*. These are **between-school** comparisons. +- **Stage 2 (individual level):** within each treatment school, students are randomized to chatbot access. This is a **within-school** comparison. +- + +| Estimand | Randomized at | School FE? | +|------------------------|------------------------|------------------------| +| CSE vs control; debiasing vs CSE | school level | **No** — absorbed by FE; analyzed between-school | +| Chatbot direct effect | individual, within school | **Yes** — FE removes between-school noise (precision gain) | +| Spillover (school's treated share) | school level (one value per school) | **No** — absorbed by FE; identified between schools | + +------------------------------------------------------------------------ + +# Baird et al. 2018 + +The key result: **the power to detect the average (pooled) treatment effect declines exactly as you gain the ability to identify spillover and slope effects.** You cannot maximize both. + +Rules they derive: + +- If you want **both** the treatment and spillover effects, put **more clusters at the extreme saturations** (e.g. more schools at 20% and 80% than at 40/60%). The needed skew shrinks as the ICC rises. +- If you only want the **slope** (how spillovers change with exposure), you don't need a pure-control group; use extreme symmetric saturations (\~0.1 and 0.9). +- For the **pooled treatment effect**, a simple partial-population design (pure control + one interior saturation) is optimal; spreading saturations *reduces* its power. +- **Bottom line:** if spillovers are likely small and ICC is high, don't saturate. + +------------------------------------------------------------------------ + +# Tool 1 - Baird et al. analytic MDEs (base R) + +These are the authors' published functions (base R, no packages) + +direct file: + +full software page: + +`power_slope()` gives the **minimum detectable spillover slope**; how precisely we can estimate the way spillovers change across saturations. + +```{r baird-functions} +# Source: Baird, Bohren, McIntosh & Ozler -- PDEL "R Implementation.R" +# https://pdel.ucsd.edu/_files/R%20Implementation.R +# Base R only (uses qt()); no packages required. Arguments explained in the table below. + +power_slope <- function(n, C, alpha, gamma, tau, sigma, pi, f, j, k) { + t_alpha <- qt(1 - alpha/2, n*C - 3) + t_gamma <- qt(gamma, n*C - 3) + mu_ind <- pi * f # E[treated] per saturation bin + p_ind <- (1 - pi) * f # E[untreated] per saturation bin + varN <- tau + sigma # total individual variance + varCo <- (n - 1) * tau # cluster-correlation contribution + Var_T <- (varCo*(1/f[j] + 1/f[k]) + varN*(1/mu_ind[j] + 1/mu_ind[k])) / (n*C) + Var_S <- (varCo*(1/f[j] + 1/f[k]) + varN*(1/p_ind[j] + 1/p_ind[k])) / (n*C) + list(MDSE_T = ((t_alpha + t_gamma)/(pi[k] - pi[j])) * sqrt(Var_T), # treatment slope + MDSE_S = ((t_alpha + t_gamma)/(pi[k] - pi[j])) * sqrt(Var_S)) # spillover slope +} + +# power_pooled(): pooled treatment effect vs a PURE CONTROL (MDE_Tonly) and the +# pooled SPILLOVER LEVEL on untreated students vs a pure control (MDE_S). +# Requires a saturation-0 ("pure control") arm +power_pooled <- function(n, C, alpha, gamma, tau, sigma, pi, f) { + t_alpha <- qt(1 - alpha/2, n*C - 3); t_gamma <- qt(gamma, n*C - 3) + mu <- sum(pi * f); eta <- sum(pi^2 * f) + psi <- if (pi[1] == 0) f[1] else 0 # share of schools that are pure control + muS <- 1 - mu - psi + etaT <- (eta - mu^2)/(1 - psi) - (psi/(1 - psi)^2)*mu^2 + varN <- tau + sigma; varCo <- (n - 1)*tau + Var <- 1/(n*C)*(varCo*(1/(psi*(1-psi)) + (1-psi)/(mu ^2)*etaT) + varN*(psi+mu )/(mu *psi)) + VarS <- 1/(n*C)*(varCo*(1/(psi*(1-psi)) + (1-psi)/(muS^2)*etaT) + varN*(psi+muS)/(muS*psi)) + Var_T <- 1/(n*C)*(varCo*(eta-mu^2)/(mu^2*(1-mu)^2) + varN/(mu*(1-mu))) + list(MDE_T = (t_alpha+t_gamma)*sqrt(Var), # pooled treatment (needs pure control) + MDE_S = (t_alpha+t_gamma)*sqrt(VarS), # pooled spillover LEVEL vs control + MDE_Tonly = (t_alpha+t_gamma)*sqrt(Var_T)) # treatment vs control +} + +# Spillover-slope MDE only (safe when a bin is a pure control: avoids the +# treated-slope term, which divides by zero when a bin has no treated students). +spill_slope_S <- function(n, C, alpha, gamma, tau, sigma, pi, f, j, k) { + tt <- qt(1 - alpha/2, n*C - 3) + qt(gamma, n*C - 3) + p_ind <- (1 - pi) * f + varN <- tau + sigma; varCo <- (n - 1)*tau + Var_S <- (varCo*(1/f[j] + 1/f[k]) + varN*(1/p_ind[j] + 1/p_ind[k])) / (n*C) + (tt / (pi[k] - pi[j])) * sqrt(Var_S) +} +``` + +The spillover effect can differ across saturation levels; its *slope* is (spillover at high saturation − spillover at low saturation) ÷ (high − low share). `power_slope()` returns the smallest such slope we could distinguish from zero: a standard-error term `sqrt(Var_S)`, scaled up by the test-and-power constants `(t_alpha + t_gamma)`, and **divided by the saturation gap** `(pi[k] − pi[j])`. + +## Design comparison: equal-thirds vs extreme-weighted + +```{r design-comparison} +library(knitr) +library(kableExtra) # install.packages("kableExtra") if needed + +n <- 90; C <- 450 +outcomes <- list(c("Pregnancy rate", 0.050), + c("Overconfidence score", 0.065), + c("Depression / Knowledge", 0.100), + c("Test scores", 0.200)) + +cmp <- do.call(rbind, lapply(outcomes, function(o) { + icc <- as.numeric(o[2]); tau <- icc; sig <- 1 - icc + eq <- power_slope(n, C, .05, .80, tau, sig, c(.25,.50,.75), c(1/3,1/3,1/3), 1, 3)$MDSE_S + ex <- power_slope(n, C, .05, .80, tau, sig, c(.20,.50,.80), c(.40,.20,.40), 1, 3)$MDSE_S + data.frame(Outcome = o[1], + ICC = sprintf("%.3f", icc), + Eq = sprintf("%.4f", eq), + Ex = sprintf("%.4f", ex), + Gain = sprintf("-%.1f%%", (1 - ex/eq) * 100), + check.names = FALSE) +})) +colnames(cmp) <- c("Outcome", "ICC", "Equal thirds 25/50/75", + "Extreme 20/50/80 (40/20/40)", "MDE reduction") + +kable(cmp, align = c("l", "c", "c", "c", "c"), + caption = paste0("Minimum detectable spillover SLOPE (SD units) by saturation design ", + "(80% power, alpha = 0.05, 450 schools x 90 students)")) |> + kable_styling(bootstrap_options = c("striped", "hover", "condensed"), + full_width = FALSE) |> + add_header_above(c(" " = 2, "MDE of spillover slope (SD)" = 2, " " = 1)) |> + column_spec(4, bold = TRUE) +``` + +The extreme-weighted design (20/50/80, with 40% of schools at each extreme and only 20% at 50%) lowers the detectable spillover by roughly a fifth to a quarter across outcomes. Even so, the detectable spillover is large in absolute terms (about 0.14-0.31 SD) and worst for high-ICC outcomes like test scores, so only sizeable spillovers will be detectable. + +## Pure control group addition + +In the current design no treatment school is at saturation 0 but we can add one (CSE schools where 0% get the chatbot). Doing so is what unlocks Baird's pure-control formulas and gives two *spillover-clean* comparisons. + +| \# | Estimand | What it answers | Tool | Needs pure control? | +|---------------|---------------|---------------|---------------|---------------| +| 1 | **Direct effect (within-FE)** | chatbot vs a *classmate* without it | within-school FE formula | no | +| 2 | **Treatment vs control** | chatbot vs a *pure-control school* (spillover-clean) | `power_pooled()` `MDE_Tonly` | **yes** | +| 3 | **Spillover slope** | how spillover *changes* with saturation | `power_slope()` `MDSE_S` | no (needs ≥2 saturations) | +| 4 | **Spillover level vs control** | untreated-in-treated-school vs pure control | `power_pooled()` `MDE_S` | **yes** | + +Estimand 1 is precise but spillover-biased; estimand 2 is the clean version (needs a pure control). Estimand 3 measures the *shape* of spillover across saturations; estimand 4 measures its *level* against a clean baseline. + +For estimand 1 the formula is the within-school one, `(t_alpha + t_gamma) * sigma_within * sqrt(1 / (N * mean[s(1-s)]))`; the design enters only through `mean[s(1-s)]`, largest at a 50/50 split. + +## Variation across designs + +```{r master-menu} +icc <- 0.20 # test scores (highest ICC = hardest); edit to compare outcomes +zc <- qnorm(0.975) + qnorm(0.80) # 2.80 test+power constant +Ntot <- 90 * 450 + +# Estimand 1: within-school direct effect (chatbot vs classmate). 0 if no within-school variation. +direct_mde <- function(pi, f) { + mean_s <- sum(f * pi * (1 - pi)) # avg treatment-assignment variance + if (mean_s == 0) return(NA) + zc * sqrt(1 - icc) * sqrt(1 / (Ntot * mean_s)) +} + +# design menu: name, saturations (pi), school shares (f), spillover-slope bins (j,k), pure control? +menu <- list( + list("Uniform 50/50", c(.50), c(1), NA, FALSE), + list("Partial pop: control + 50%", c(0,.50), c(.50,.50), c(1,2), TRUE), + list("Equal thirds 25/50/75", c(.25,.50,.75), c(1/3,1/3,1/3), c(1,3), FALSE), + list("Extreme 20/50/80 (40/20/40)", c(.20,.50,.80), c(.40,.20,.40), c(1,3), FALSE), + list("Symmetric 20/80 (50/50)", c(.20,.80), c(.50,.50), c(1,2), FALSE), + list("Wide 10/50/90 (40/20/40)", c(.10,.50,.90), c(.40,.20,.40), c(1,3), FALSE), + list("Baird ladder 0/25/50/75/100", c(0,.25,.50,.75,1), rep(.2,5), c(2,4), TRUE), + list("Control + thirds 0/25/50/75", c(0,.25,.50,.75), rep(.25,4), c(2,4), TRUE), + list("Control + 20/50/80 (0/20/50/80)", c(0,.20,.50,.80), rep(.25,4), c(2,4), TRUE)) + +fmt <- function(x) ifelse(is.na(x) | !is.finite(x), "—", sprintf("%.3f", x)) +menu_tbl <- do.call(rbind, lapply(menu, function(d) { + pi <- d[[2]]; f <- d[[3]]; jk <- d[[4]]; pc <- d[[5]] + slope <- if (length(pi) > 1) spill_slope_S(90, 450, .05, .80, icc, 1 - icc, pi, f, jk[1], jk[2]) else NA + pooled <- if (pc) power_pooled(90, 450, .05, .80, icc, 1 - icc, pi, f) else list(MDE_Tonly = NA, MDE_S = NA) + data.frame(Design = d[[1]], `Pure control?` = ifelse(pc, "Yes", "No"), + `1. Direct (within-FE)` = fmt(direct_mde(pi, f)), + `2. Treatment vs control` = ifelse(pc, fmt(pooled$MDE_Tonly), "—"), + `3. Spillover slope` = fmt(slope), + `4. Spillover vs control` = ifelse(pc, fmt(pooled$MDE_S), "—"), + check.names = FALSE) +})) +kable(menu_tbl, align = c("l","c","c","c","c","c"), + caption = "Test scores (ICC 0.20): MDE (SD) for each estimand, by design. Lower = more power; — = not identified.") |> + kable_styling(bootstrap_options = c("striped","hover","condensed"), full_width = FALSE) |> + add_header_above(c(" " = 2, "Treatment effect" = 2, "Spillover" = 2)) |> + footnote(general = paste0( + "Estimand 1 (within-FE direct): chatbot vs a classmate without it — precise but spillover-biased. ", + "Estimand 2 (Baird power_pooled MDE_Tonly): chatbot vs a PURE-CONTROL school — spillover-clean. ", + "Estimand 3 (Baird power_slope MDSE_S): how spillover changes across saturations. ", + "Estimand 4 (Baird power_pooled MDE_S): untreated-in-treated-school vs pure control — the spillover LEVEL. ", + "Estimands 2 and 4 require a saturation-0 (pure-control) arm; hence '—' for designs without one."), + general_title = "Notes: ") +``` + +1. **Direct effect (col 1)** is best under **uniform 50/50** (\~0.025) and gets worse as we add pure-control or extreme schools (they have little/no within-school chatbot variation). But this estimate is *spillover-biased*. +2. **Treatment vs control (col 2)** is the *spillover-clean* direct effect — available only with a pure control (\~0.085). It's less precise than col 1 (it's a between-school comparison) but immune to contamination. Note it's still a small, detectable effect. +3. **Spillover slope (col 3)** is best when the two extreme bins are wide *and* hold many schools: wide 10/50/90 is best (\~0.19), then symmetric (\~0.21) and extreme (\~0.24); equal thirds is weaker (\~0.31); and the ladder (\~0.40) and control+thirds (\~0.36) are the worst — spreading schools across many bins leaves only a few in each compared bin, which hurts the slope. +4. **Spillover level vs control (col 4)** is the best-powered way to detect a spillover at all (\~0.12 with the partial-population design) *because* it leans on the clean pure-control baseline. + +Columns 1 and 3 pull in opposite directions (Baird's central result): spreading toward the extremes helps the spillover but costs direct-effect precision. But if the direct effect has some slack (it stays \~0.025–0.085 SD everywhere) then the **spillover is the binding constraint**. Onme straightforward way in which case would be to **add a pure-control arm** (estimand 4, \~0.12 SD), which *also* buys a clean treatment effect (estimand 2). + +The same rankings hold for the lower-ICC outcomes (pregnancy, overconfidence); the absolute MDEs just shrink. + +------------------------------------------------------------------------ + +# Tool 2 - `RCT2` cross-check (R package) + +Its `Calsamplesize()` returns the **number of clusters** needed to detect a given effect at a target power, per assignment mechanism. The effect-size argument `mu` is *"the largest **direct** effect across treatment assignment mechanisms.* + +We run `Calsamplesize` for the saturation designs and compare the clusters it needs against our **450 schools** + +```{r rct2-inputs} +rct2_inputs <- data.frame( + Input = c("Clusters (schools)", "Students per school", "ICC", + "Direct effect to detect (mu)", "Significance / power", + "Designs compared", "Excluded", "Method"), + Value = c("450", "90", "0.10", "0.05 SD", "0.05 / 0.80", + "the 3-mechanism designs: equal thirds 25/50/75; extreme 20/50/80; wide 10/50/90", + "uniform 50/50 (1 mechanism) and symmetric 20/80 (2 mechanisms) - Calsamplesize needs 3 arms; also 0%/100% arms (direct effect undefined there)", + "RCT2 design-based (Jiang-Imai-Malani 2023); data simulated to the assumed ICC"), + check.names = FALSE) +knitr::kable(rct2_inputs, caption = "RCT2 cross-check: inputs and assumptions") |> + kableExtra::kable_styling(bootstrap_options = c("striped","hover","condensed"), full_width = FALSE) +``` + +```{r rct2-designs, eval=FALSE} +library(RCT2); library(kableExtra) +set.seed(2026) +n_per <- 90; n_clust <- 450; icc <- 0.10 +mu <- 0.05 # DIRECT effect to detect (SD); RCT2's 'mu' + +# NOTE: RCT2's Calsamplesize is built around a THREE-mechanism design (its worked +# example uses three arms) and its internal indexing errors for 1- or 2-mechanism +# designs. So we compare only the 3-mechanism designs here (uniform = 1 arm and +# symmetric 20/80 = 2 arms are excluded), and it sizes the DIRECT effect, so 0%/100% +# arms don't apply either. For each design the three ASSIGNMENT MECHANISMS are the +# three treated shares a school can be assigned to. +designs_rct2 <- list( + "Equal thirds 25/50/75" = list(lv = c(.25,.50,.75), w = c(1/3,1/3,1/3)), # mechanisms: 25% / 50% / 75% treated + "Extreme 20/50/80" = list(lv = c(.20,.50,.80), w = c(.40,.20,.40)), # mechanisms: 20% / 50% / 80% treated + "Wide 10/50/90" = list(lv = c(.10,.50,.90), w = c(.40,.20,.40))) # mechanisms: 10% / 50% / 90% treated + +build_dat <- function(lv, w) { + sat <- sample(lv, n_clust, replace = TRUE, prob = w) # random mechanism per school (matches the run that worked) + A <- rep(sat, each = n_per) # A = the school's assignment mechanism (treated share) + Z <- rbinom(n_clust * n_per, 1, A) # treat each student at the school's rate + u <- rep(rnorm(n_clust, 0, sqrt(icc)), each = n_per) + e <- rnorm(n_clust * n_per, 0, sqrt(1 - icc)) + Y <- 0.10 * Z + 0.10 * A * (1 - Z) + u + e + data.frame(Z = Z, A = A, Y = Y, id = rep(1:n_clust, each = n_per)) # integer id (matches the run that worked) +} + +rows <- lapply(names(designs_rct2), function(nm) { + d <- designs_rct2[[nm]] + dat <- build_dat(d$lv, d$w) + qa <- d$w / sum(d$w) # share of schools per mechanism + ss <- Calsamplesize(dat, mu, qa, alpha = 0.05, beta = 0.20) # class "sample": numeric vector + cl <- as.numeric(ss) # clusters needed, one per mechanism + data.frame(Design = nm, + Mechanisms = paste0(d$lv * 100, "%", collapse = " / "), + `Clusters needed (per mechanism)` = paste(round(cl), collapse = " / "), + `Min clusters (best mechanism)` = round(min(cl)), + check.names = FALSE) +}) +kable(do.call(rbind, rows), align = c("l","c","c","c"), + caption = "RCT2 (design-based): clusters needed to detect a 0.05 SD DIRECT effect, by saturation design (ICC 0.10)") |> + kable_styling(bootstrap_options = c("striped","hover","condensed"), full_width = FALSE) |> + footnote(general = paste0( + "Calsamplesize sizes the DIRECT effect (mu = largest direct effect across mechanisms). ", + "Compare 'Min clusters' against our 450 schools; the 50% mechanism is the most efficient. ", + "Numbers are randomization-based and corroborate Baird directionally, not exactly."), + general_title = "Notes: ") +``` + +------------------------------------------------------------------------ + +# Design × spillover grid (simulation) + +For each assignment design, and a range of assumed spillover sizes (**including none**), this simulation reports (a) the **bias** of the naive within-school direct-effect estimate, (b) whether the true direct effect can be **recovered**, and (c) the **power to detect the spillover**. + +## Assumptions + +- **Design held fixed:** 450 treatment schools, 90 surveyed students each. +- **Outcome standardized** so total variance = 1; the **ICC** sets the between-school share of variance (we use 0.10 here). +- **True direct chatbot effect planted at 0.10 SD** +- **Spillover model:** untreated students gain `spillover * (school's treated share)` —{0, 0.10, 0.20, 0.30} SD; **0 = no spillover** (a validity check). +- **Nine designs:** uniform 50/50; partial population (0/50); equal thirds (25/50/75); extreme (20/50/80); symmetric (20/80); wide (10/50/90); control+full (0/50/100); control+thirds (0/25/50/75); and the Baird ladder (0/25/50/75/100). +- **What we measure per design × spillover:** (i) the **bias** of the naive within-school estimate (the attenuation a design would suffer), and (ii) the **power to detect the spillover** by testing whether *untreated* students' outcomes rise with their school's saturation. + +```{r grid-setup} +library(fixest) # fixed-effects regressions + cluster-robust SEs +library(kableExtra) # formatted table output +set.seed(2026) + +S <- 450 # treatment schools +n_per <- 90 # students surveyed per school +icc <- 0.10 # between-school share of variance (ICC) +b2_true <- 0.10 # TRUE direct chatbot effect we plant (SD units) +spills <- c(0, 0.10, 0.20, 0.30) # spillover sizes to test (0 = none) +REPS <- 150 # reps per cell + +# every candidate design as (saturation levels, share of schools at each level) +DESIGN_SPECS <- list( + "Uniform 50/50" = list(lv = c(.50), w = c(1)), + "Partial pop 0/50" = list(lv = c(0,.50), w = c(.50,.50)), + "Equal thirds 25/50/75" = list(lv = c(.25,.50,.75), w = c(1/3,1/3,1/3)), + "Extreme 20/50/80" = list(lv = c(.20,.50,.80), w = c(.40,.20,.40)), + "Symmetric 20/80" = list(lv = c(.20,.80), w = c(.50,.50)), + "Wide 10/50/90" = list(lv = c(.10,.50,.90), w = c(.40,.20,.40)), + "Control+full 0/50/100" = list(lv = c(0,.50,1), w = c(1/3,1/3,1/3)), + "Control+thirds 0/25/50/75" = list(lv = c(0,.25,.50,.75), w = c(.25,.25,.25,.25)), + "Baird ladder 0/25/50/75/100" = list(lv = c(0,.25,.50,.75,1), w = rep(.2, 5)), + "Control + 20/50/80 0/20/50/80" = list(lv = c(0,.20,.50,.80), w = c(.25,.25,.25,.25))) +designs <- names(DESIGN_SPECS) +``` + +## Step 1 — assign each school a saturation + +`assign_sat()` gives every school its treated share under the chosen design, stratified by CSE arm so the two arms stay balanced. + +```{r grid-step1} +assign_sat <- function(design, deb) { + sp <- DESIGN_SPECS[[design]]; lv <- sp$lv; w <- sp$w + if (length(lv) == 1) return(rep(lv, S)) # uniform: every school same share + sat <- numeric(S) + for (arm in c(0, 1)) { # stratify within each CSE arm + idx <- which(deb == arm); m <- length(idx) + cnt <- floor(w * m); cnt[length(cnt)] <- m - sum(cnt[-length(cnt)]) + lvl <- unlist(mapply(function(c, l) rep(l, c), cnt, lv)) + sat[idx] <- sample(lvl) # shuffle which school gets which + } + sat +} +``` + +## Step 2 — generate virtual study and estimate it two ways + +The outcome is built as: + +``` +Y = school_effect(ICC) + 0.10*Chatbot + spillover*share*(1 - Chatbot) + noise +``` + +i.e. a school baseline + the planted direct effect for treated students + a spillover bonus for *untreated* students that grows with their school's treated share + individual noise. We then run both regressions on that one dataset. + +```{r grid-step2} +sim_once <- function(design, spill) { + deb <- sample(rep(c(0, 1), each = S / 2)) # half schools debiasing + sat <- assign_sat(design, deb) # Step 1 + k <- round(sat * n_per) # treated count per school + Tt <- unlist(lapply(k, function(kk) sample(rep(c(1, 0), c(kk, n_per - kk))))) + d <- data.frame(school = rep(seq_len(S), each = n_per), + s_s = rep(sat, each = n_per), T = Tt) # s_s = school treated share + u <- rep(rnorm(S, 0, sqrt(icc)), each = n_per) # between-school (ICC) + e <- rnorm(S * n_per, 0, sqrt(1 - icc)) # within-school + d$y <- u + b2_true * d$T + spill * d$s_s * (1 - d$T) + e # plant the outcome + + # (i) NAIVE within-school direct effect (mixed schools only): biased if spillover present + mid <- d[d$s_s > 0 & d$s_s < 1, ] + b2_fe <- coef(feols(y ~ T | school, data = mid, cluster = ~school))[["T"]] + + # (ii) SPILLOVER detection (robust for every design): do UNTREATED students' + # outcomes rise with their school's saturation? (cluster-robust by school) + un <- d[d$T == 0, ] + reject <- NA_real_ + if (length(unique(un$s_s)) > 1) { + m <- feols(y ~ s_s, data = un, cluster = ~school) + reject <- as.numeric(abs(m$coeftable["s_s", "t value"]) > 1.96) + } + c(b2_fe = b2_fe, reject = reject) +} +``` + +## Step 3 — repeat over the whole grid and summarise + +```{r grid-step3} +res <- data.frame() +for (dz in designs) for (sp in spills) { + draws <- replicate(REPS, sim_once(dz, sp)) # 2 x REPS matrix + res <- rbind(res, data.frame( + design = dz, + spill = sp, + est = mean(draws["b2_fe", ]), # naive direct estimate (true = 0.10) + power = mean(draws["reject", ], na.rm = TRUE))) # power to detect spillover +} +res +``` + +## Step 4 — format the results table + +```{r grid-step4} +disp <- data.frame( + `Spillover (delta)` = sprintf("%.2f", res$spill), + `Naive direct estimate` = sprintf("%+.3f", res$est), + `Spillover detection power` = ifelse(is.finite(res$power), sprintf("%.0f%%", 100 * res$power), "—"), + check.names = FALSE) + +kt <- kable(disp, align = c("c", "c", "c"), + caption = sprintf(paste0("Design x spillover grid (live simulation): ", + "true direct effect = %.2f SD, ICC = %.2f, %d reps/cell"), + b2_true, icc, REPS)) |> + kable_styling(bootstrap_options = c("striped", "hover", "condensed"), full_width = FALSE) +ns <- length(spills) +for (i in seq_along(designs)) kt <- pack_rows(kt, designs[i], (i - 1)*ns + 1, i*ns) +kt |> footnote(general = paste0( + "Naive direct estimate = within-school FE estimate on mixed schools; the TRUE value is 0.10 but it ", + "is biased by about -0.5*spillover under EVERY design (it collapses to ~0 at spillover 0.20 and flips ", + "negative at 0.30).", + "Spillover detection power = share of reps where untreated students' outcomes significantly rise with ", + "their school's saturation (cluster-robust, alpha = 0.05). Uniform 50/50 has no variation among the ", + "untreated, so it cannot detect spillover ('—'). The delta = 0 rows are validity checks (~5%). "), + general_title = "Notes: ") +``` + +- **Spillover detection power rises with the spillover size**. At a 0.20 spillover, the **spread designs do best** (symmetric 20/80 and wide 10/50/90 ≈ 90%+, control+thirds and partial-pop ≈ 85–90%, extreme ≈ 85%), while **equal thirds (\~68%)** and **control+full 0/50/100 (\~70%)** trail; the latter because its 100%-saturation third has no untreated students to measure spillover on. + +------------------------------------------------------------------------ + +# TBD: + +- **Do we want to estimate spillover effects (is that a study objective)?** + - **If yes:** a saturation design is justified and it should probably include a **pure-control (0%) arm plus several interior levels**. + + - **If no:** don't saturate. + +------------------------------------------------------------------------ + +# Scorecard: both tools at a glance + +```{r scorecard} +ref_spill <- 0.20 +zc <- qnorm(.975) + qnorm(.80) +score <- do.call(rbind, lapply(designs, function(dz) { + sp <- DESIGN_SPECS[[dz]] + meanS <- sum(sp$w * sp$lv * (1 - sp$lv)) # avg treatment-assignment variance + me <- if (meanS > 0) zc * sqrt(1 - icc) * sqrt(1/(S*n_per*meanS)) else NA + pw <- res$power[res$design == dz & res$spill == ref_spill] + data.frame(Design = dz, + `Pure control?` = ifelse(0 %in% sp$lv, "Yes", "No"), + `Main-effect MDE` = ifelse(is.na(me), "—", sprintf("%.3f SD", me)), + `Spillover power (d=0.20)` = ifelse(length(pw) && is.finite(pw), sprintf("%.0f%%", 100*pw), "—"), + check.names = FALSE) +})) +kable(score, align = c("l","c","c","c"), + caption = sprintf("Design scorecard (ICC %.2f): main-effect precision (Tool 1) vs spillover detection (grid sim)", icc)) |> + kable_styling(bootstrap_options = c("striped","hover","condensed"), full_width = FALSE) |> + add_header_above(c(" " = 2, "Tool 1 (analytic)" = 1, "Grid (simulation)" = 1)) |> + footnote(general = paste0( + "Main-effect MDE: within-school FE formula (Tool 1) - smaller = more precise direct effect. ", + "Spillover power: grid simulation at a true 0.20 SD spillover - higher = easier to detect. ", + "The columns trade off: spread/extreme designs detect spillover best at a slightly larger main-effect ", + "MDE; uniform 50/50 has the smallest main-effect MDE but cannot detect spillover. A pure-control arm ", + "additionally gives a spillover-CLEAN direct effect (Tool 1 master menu)."), + general_title = "Notes: ") +``` + +**Uniform 50/50** has the smallest main-effect MDE (most precise direct effect) but **cannot detect spillover at all**. The **spread designs** (symmetric 20/80 and wide 10/50/90 \~94%, extreme \~86%) and the better **pure-control designs** (partial-pop \~87%, control+thirds \~89%) detect spillover best, for only a small increase in the main-effect MDE. + +------------------------------------------------------------------------ + +# References + +- Baird, S., Bohren, J. A., McIntosh, C., & Özler, B. (2018). *Optimal Design of Experiments in the Presence of Interference.* Review of Economics and Statistics, 100(5), 844–860. +- Bohren, Staples, Baird, McIntosh, Özler (2016). *Power Calculation Software for Randomized Saturation Experiments* (PDEL, UCSD) — GUI + Python/R/MATLAB code: +- Jiang, Z., Imai, K., & Malani, A. (2023). *Statistical Inference and Power Analysis for Direct and Spillover Effects in Two-Stage Randomized Experiments.* Biometrics, 79(3), 2370–2383. `RCT2` package: +- World Bank Development Impact blog (practical summary): \`\`\` diff --git a/cse-dr/powercalc_NG_approach/spillover_power_check.html b/cse-dr/powercalc_NG_approach/spillover_power_check.html new file mode 100644 index 0000000..bd09e45 --- /dev/null +++ b/cse-dr/powercalc_NG_approach/spillover_power_check.html @@ -0,0 +1,4268 @@ + + + + + + + + + + + + + + + +Spillover Power & the Saturation-Design + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + +
+
+
+
+
+ +
+ + + + + + + +
+

1 Purpose of this +document

+
    +
  1. Shows, with two independent tools, how much power we have to +detect a spillover under different saturation schemes: +
      +
    • the Baird et al. analytic MDE formulas (their +published R code), and
    • +
    • the RCT2 package (Jiang, Imai & +Malani 2023)
    • +
  2. +
+
+
+
+

2 Inputs, notation, and +assumptions

+
library(knitr); library(kableExtra)
+inputs <- data.frame(
+  Parameter = c("Treatment schools (C)", "Students per school (n)", "Total students",
+                "Significance (alpha)", "Power (gamma)",
+                "ICC — pregnancy", "ICC — overconfidence",
+                "ICC — depression / knowledge", "ICC — test scores",
+                "Pregnancy base rate (1 yr / 4 yr)", "Overconfidence mean / SD",
+                "Female share (girls-only rows)"),
+  Value = c("450", "90", "40,500", "0.05 (two-sided)", "0.80",
+            "0.050", "0.065", "0.100", "0.200",
+            "0.056 / 0.168", "10.4 / 27.7", "0.50"),
+  Source = c("design doc", "design doc (30/grade x 3 grades)", "design doc",
+             "convention", "convention",
+             "ENHOGAR (conservative)", "2024 pilot (N = 800)",
+             "health literature / pilot", "Pruebas Nacionales",
+             "WDI / Dupas 2018", "2024 pilot", "design doc"),
+  check.names = FALSE)
+kable(inputs, caption = "Table 1. Design inputs and data (with sources)") |>
+  kable_styling(bootstrap_options = c("striped","hover","condensed"), full_width = FALSE)
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+Table 1. Design inputs and data (with sources) +
+Parameter + +Value + +Source +
+Treatment schools (C) + +450 + +design doc +
+Students per school (n) + +90 + +design doc (30/grade x 3 grades) +
+Total students + +40,500 + +design doc +
+Significance (alpha) + +0.05 (two-sided) + +convention +
+Power (gamma) + +0.80 + +convention +
+ICC — pregnancy + +0.050 + +ENHOGAR (conservative) +
+ICC — overconfidence + +0.065 + +2024 pilot (N = 800) +
+ICC — depression / knowledge + +0.100 + +health literature / pilot +
+ICC — test scores + +0.200 + +Pruebas Nacionales +
+Pregnancy base rate (1 yr / 4 yr) + +0.056 / 0.168 + +WDI / Dupas 2018 +
+Overconfidence mean / SD + +10.4 / 27.7 + +2024 pilot +
+Female share (girls-only rows) + +0.50 + +design doc +
+
assum <- data.frame(
+  Assumption = c("Spillover channel", "Spillover sizes swept", "Saturation designs",
+                 "Debiasing coding", "Estimator (chatbot effect)",
+                 "Estimator (spillover)", "Simulation reps"),
+  Choice = c("untreated outcome rises by  spillover x (school's treated share)",
+             "0, 0.10, 0.20, 0.30 SD  (0 = no spillover)",
+             "uniform 50/50; equal thirds 25/50/75; extreme 20/50/80 @ 40/20/40",
+             "centered (Deb - 0.5) so the chatbot coefficient is the pooled effect",
+             "school fixed effects + cluster-robust SEs",
+             "controls for the school share; no school FE (strata FE + cluster SEs)",
+             "300 per cell (raise for smoother power)"),
+  Type = c("our modelling choice (Baird-style)", "placeholder — replace with pilot",
+           "design options compared", "specification choice",
+           "standard", "standard", "numerical"),
+  check.names = FALSE)
+kable(assum, caption = "Table 2. Modelling assumptions (our choices)") |>
+  kable_styling(bootstrap_options = c("striped","hover","condensed"), full_width = FALSE)
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+Table 2. Modelling assumptions (our choices) +
+Assumption + +Choice + +Type +
+Spillover channel + +untreated outcome rises by spillover x (school’s treated share) + +our modelling choice (Baird-style) +
+Spillover sizes swept + +0, 0.10, 0.20, 0.30 SD (0 = no spillover) + +placeholder — replace with pilot +
+Saturation designs + +uniform 50/50; equal thirds 25/50/75; extreme 20/50/80 @ 40/20/40 + +design options compared +
+Debiasing coding + +centered (Deb - 0.5) so the chatbot coefficient is the pooled effect + +specification choice +
+Estimator (chatbot effect) + +school fixed effects + cluster-robust SEs + +standard +
+Estimator (spillover) + +controls for the school share; no school FE (strata FE + cluster SEs) + +standard +
+Simulation reps + +300 per cell (raise for smoother power) + +numerical +
+
notation <- data.frame(
+  Symbol = c("n", "C", "tau", "sigma", "varN", "varCo", "pi", "f",
+             "mu_ind", "p_ind", "j, k", "t_alpha", "t_gamma", "Var_S", "MDSE_S"),
+  Meaning = c("students per school", "number of schools",
+              "between-school variance", "within-school variance",
+              "total individual variance", "clustering penalty",
+              "saturation levels (treated share per bin)",
+              "share of schools at each saturation",
+              "expected treated share in a bin", "expected untreated share in a bin",
+              "the two saturation bins compared for the slope",
+              "test critical value", "power critical value",
+              "sampling variance of the spillover-slope estimate",
+              "minimum detectable spillover slope"),
+  `Formula / value` = c("90", "450", "= ICC", "= 1 - ICC", "tau + sigma  (= 1)",
+              "(n - 1) * tau", "e.g. c(.25,.50,.75)", "e.g. c(1/3,1/3,1/3)",
+              "pi * f", "(1 - pi) * f", "the extreme bins (1 and 3)",
+              "qt(1 - alpha/2, nC - 3)", "qt(gamma, nC - 3)",
+              "[varCo*(1/f_j+1/f_k) + varN*(1/p_j+1/p_k)] / (nC)",
+              "(t_alpha + t_gamma) * sqrt(Var_S) / (pi_k - pi_j)"),
+  Source = c("design", "design", "definition (= ICC)", "definition",
+             "Baird et al.", "Baird et al.", "design choice", "design choice",
+             "Baird et al.", "Baird et al.", "our choice",
+             "Baird et al.", "Baird et al.", "Baird et al.", "Baird et al."),
+  check.names = FALSE)
+kable(notation, caption = "Table 3. Notation used in the formulae") |>
+  kable_styling(bootstrap_options = c("striped","hover","condensed"), full_width = FALSE) 
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+Table 3. Notation used in the formulae +
+Symbol + +Meaning + +Formula / value + +Source +
+n + +students per school + +90 + +design +
+C + +number of schools + +450 + +design +
+tau + +between-school variance + += ICC + +definition (= ICC) +
+sigma + +within-school variance + += 1 - ICC + +definition +
+varN + +total individual variance + +tau + sigma (= 1) + +Baird et al.  +
+varCo + +clustering penalty + +(n - 1) * tau + +Baird et al.  +
+pi + +saturation levels (treated share per bin) + +e.g. c(.25,.50,.75) + +design choice +
+f + +share of schools at each saturation + +e.g. c(1/3,1/3,1/3) + +design choice +
+mu_ind + +expected treated share in a bin + +pi * f + +Baird et al.  +
+p_ind + +expected untreated share in a bin + +(1 - pi) * f + +Baird et al.  +
+j, k + +the two saturation bins compared for the slope + +the extreme bins (1 and 3) + +our choice +
+t_alpha + +test critical value + +qt(1 - alpha/2, nC - 3) + +Baird et al.  +
+t_gamma + +power critical value + +qt(gamma, nC - 3) + +Baird et al.  +
+Var_S + +sampling variance of the spillover-slope estimate + +[varCo(1/f_j+1/f_k) + varN(1/p_j+1/p_k)] / (nC) + +Baird et al.  +
+MDSE_S + +minimum detectable spillover slope + +(t_alpha + t_gamma) * sqrt(Var_S) / (pi_k - pi_j) + +Baird et al.  +
+
+
+
+

3 Background: the design +and why spillovers matter

+

The chatbot is randomized in two stages:

+
    +
  • Stage 1 (school level): schools are assigned to +Regular CSE vs CSE + Debiasing. These are +between-school comparisons.
  • +
  • Stage 2 (individual level): within each treatment +school, students are randomized to chatbot access. This is a +within-school comparison.
  • +
  • +
+ +++++ + + + + + + + + + + + + + + + + + + + + + + + + +
EstimandRandomized atSchool FE?
CSE vs control; debiasing vs CSEschool levelNo — absorbed by FE; analyzed between-school
Chatbot direct effectindividual, within schoolYes — FE removes between-school noise (precision +gain)
Spillover (school’s treated share)school level (one value per school)No — absorbed by FE; identified between +schools
+
+
+
+

4 Baird et al. 2018

+

The key result: the power to detect the average (pooled) +treatment effect declines exactly as you gain the ability to identify +spillover and slope effects. You cannot maximize both.

+

Rules they derive:

+
    +
  • If you want both the treatment and spillover +effects, put more clusters at the extreme saturations +(e.g. more schools at 20% and 80% than at 40/60%). The needed skew +shrinks as the ICC rises.
  • +
  • If you only want the slope (how spillovers change +with exposure), you don’t need a pure-control group; use extreme +symmetric saturations (~0.1 and 0.9).
  • +
  • For the pooled treatment effect, a simple +partial-population design (pure control + one interior saturation) is +optimal; spreading saturations reduces its power.
  • +
  • Bottom line: if spillovers are likely small and ICC +is high, don’t saturate.
  • +
+
+
+
+

5 Tool 1 - Baird et +al. analytic MDEs (base R)

+

These are the authors’ published functions (base R, no packages)

+

direct file: https://pdel.ucsd.edu/_files/R%20Implementation.R

+

full software page: https://pdel.ucsd.edu/about/tools/index.html

+

power_slope() gives the minimum detectable +spillover slope; how precisely we can estimate the way +spillovers change across saturations.

+
# Source: Baird, Bohren, McIntosh & Ozler -- PDEL "R Implementation.R"
+#   https://pdel.ucsd.edu/_files/R%20Implementation.R
+# Base R only (uses qt()); no packages required. Arguments explained in the table below.
+
+power_slope <- function(n, C, alpha, gamma, tau, sigma, pi, f, j, k) {
+  t_alpha <- qt(1 - alpha/2, n*C - 3)
+  t_gamma <- qt(gamma, n*C - 3)
+  mu_ind <- pi * f                 # E[treated]   per saturation bin
+  p_ind  <- (1 - pi) * f           # E[untreated] per saturation bin
+  varN  <- tau + sigma             # total individual variance
+  varCo <- (n - 1) * tau           # cluster-correlation contribution
+  Var_T <- (varCo*(1/f[j] + 1/f[k]) + varN*(1/mu_ind[j] + 1/mu_ind[k])) / (n*C)
+  Var_S <- (varCo*(1/f[j] + 1/f[k]) + varN*(1/p_ind[j]  + 1/p_ind[k]))  / (n*C)
+  list(MDSE_T = ((t_alpha + t_gamma)/(pi[k] - pi[j])) * sqrt(Var_T),   # treatment slope
+       MDSE_S = ((t_alpha + t_gamma)/(pi[k] - pi[j])) * sqrt(Var_S))   # spillover slope
+}
+
+# power_pooled(): pooled treatment effect vs a PURE CONTROL (MDE_Tonly) and the
+# pooled SPILLOVER LEVEL on untreated students vs a pure control (MDE_S).
+# Requires a saturation-0 ("pure control") arm
+power_pooled <- function(n, C, alpha, gamma, tau, sigma, pi, f) {
+  t_alpha <- qt(1 - alpha/2, n*C - 3); t_gamma <- qt(gamma, n*C - 3)
+  mu  <- sum(pi * f); eta <- sum(pi^2 * f)
+  psi <- if (pi[1] == 0) f[1] else 0          # share of schools that are pure control
+  muS <- 1 - mu - psi
+  etaT <- (eta - mu^2)/(1 - psi) - (psi/(1 - psi)^2)*mu^2
+  varN <- tau + sigma; varCo <- (n - 1)*tau
+  Var  <- 1/(n*C)*(varCo*(1/(psi*(1-psi)) + (1-psi)/(mu ^2)*etaT) + varN*(psi+mu )/(mu *psi))
+  VarS <- 1/(n*C)*(varCo*(1/(psi*(1-psi)) + (1-psi)/(muS^2)*etaT) + varN*(psi+muS)/(muS*psi))
+  Var_T <- 1/(n*C)*(varCo*(eta-mu^2)/(mu^2*(1-mu)^2) + varN/(mu*(1-mu)))
+  list(MDE_T = (t_alpha+t_gamma)*sqrt(Var),       # pooled treatment (needs pure control)
+       MDE_S = (t_alpha+t_gamma)*sqrt(VarS),      # pooled spillover LEVEL vs control
+       MDE_Tonly = (t_alpha+t_gamma)*sqrt(Var_T)) # treatment vs control
+}
+
+# Spillover-slope MDE only (safe when a bin is a pure control: avoids the
+# treated-slope term, which divides by zero when a bin has no treated students).
+spill_slope_S <- function(n, C, alpha, gamma, tau, sigma, pi, f, j, k) {
+  tt <- qt(1 - alpha/2, n*C - 3) + qt(gamma, n*C - 3)
+  p_ind <- (1 - pi) * f
+  varN <- tau + sigma; varCo <- (n - 1)*tau
+  Var_S <- (varCo*(1/f[j] + 1/f[k]) + varN*(1/p_ind[j] + 1/p_ind[k])) / (n*C)
+  (tt / (pi[k] - pi[j])) * sqrt(Var_S)
+}
+

The spillover effect can differ across saturation levels; its +slope is (spillover at high saturation − spillover at low +saturation) ÷ (high − low share). power_slope() returns the +smallest such slope we could distinguish from zero: a standard-error +term sqrt(Var_S), scaled up by the test-and-power constants +(t_alpha + t_gamma), and divided by the saturation +gap (pi[k] − pi[j]).

+
+

5.1 Design comparison: +equal-thirds vs extreme-weighted

+
library(knitr)
+library(kableExtra)   # install.packages("kableExtra") if needed
+
+n <- 90; C <- 450
+outcomes <- list(c("Pregnancy rate",         0.050),
+                 c("Overconfidence score",   0.065),
+                 c("Depression / Knowledge", 0.100),
+                 c("Test scores",            0.200))
+
+cmp <- do.call(rbind, lapply(outcomes, function(o) {
+  icc <- as.numeric(o[2]); tau <- icc; sig <- 1 - icc
+  eq <- power_slope(n, C, .05, .80, tau, sig, c(.25,.50,.75), c(1/3,1/3,1/3), 1, 3)$MDSE_S
+  ex <- power_slope(n, C, .05, .80, tau, sig, c(.20,.50,.80), c(.40,.20,.40), 1, 3)$MDSE_S
+  data.frame(Outcome = o[1],
+             ICC = sprintf("%.3f", icc),
+             Eq  = sprintf("%.4f", eq),
+             Ex  = sprintf("%.4f", ex),
+             Gain = sprintf("-%.1f%%", (1 - ex/eq) * 100),
+             check.names = FALSE)
+}))
+colnames(cmp) <- c("Outcome", "ICC", "Equal thirds 25/50/75",
+                   "Extreme 20/50/80 (40/20/40)", "MDE reduction")
+
+kable(cmp, align = c("l", "c", "c", "c", "c"),
+      caption = paste0("Minimum detectable spillover SLOPE (SD units) by saturation design ",
+                       "(80% power, alpha = 0.05, 450 schools x 90 students)")) |>
+  kable_styling(bootstrap_options = c("striped", "hover", "condensed"),
+                full_width = FALSE) |>
+  add_header_above(c(" " = 2, "MDE of spillover slope (SD)" = 2, " " = 1)) |>
+  column_spec(4, bold = TRUE) 
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+Minimum detectable spillover SLOPE (SD units) by saturation design (80% +power, alpha = 0.05, 450 schools x 90 students) +
+ +
+MDE of spillover slope (SD) +
+
+
+Outcome + +ICC + +Equal thirds 25/50/75 + +Extreme 20/50/80 (40/20/40) + +MDE reduction +
+Pregnancy rate + +0.050 + +0.1819 + +0.1428 + +-21.5% +
+Overconfidence score + +0.065 + +0.1983 + +0.1549 + +-21.9% +
+Depression / Knowledge + +0.100 + +0.2320 + +0.1799 + +-22.4% +
+Test scores + +0.200 + +0.3085 + +0.2373 + +-23.1% +
+

The extreme-weighted design (20/50/80, with 40% of schools at each +extreme and only 20% at 50%) lowers the detectable spillover by roughly +a fifth to a quarter across outcomes. Even so, the detectable spillover +is large in absolute terms (about 0.14-0.31 SD) and worst for high-ICC +outcomes like test scores, so only sizeable spillovers will be +detectable.

+
+
+

5.2 Pure control group +addition

+

In the current design no treatment school is at saturation 0 but we +can add one (CSE schools where 0% get the chatbot). Doing so is what +unlocks Baird’s pure-control formulas and gives two +spillover-clean comparisons.

+ +++++++ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#EstimandWhat it answersToolNeeds pure control?
1Direct effect (within-FE)chatbot vs a classmate without itwithin-school FE formulano
2Treatment vs controlchatbot vs a pure-control school (spillover-clean)power_pooled() MDE_Tonlyyes
3Spillover slopehow spillover changes with saturationpower_slope() MDSE_Sno (needs ≥2 saturations)
4Spillover level vs controluntreated-in-treated-school vs pure controlpower_pooled() MDE_Syes
+

Estimand 1 is precise but spillover-biased; estimand 2 is the clean +version (needs a pure control). Estimand 3 measures the shape +of spillover across saturations; estimand 4 measures its level +against a clean baseline.

+

For estimand 1 the formula is the within-school one, +(t_alpha + t_gamma) * sigma_within * sqrt(1 / (N * mean[s(1-s)])); +the design enters only through mean[s(1-s)], largest at a +50/50 split.

+
+
+

5.3 Variation across +designs

+
icc  <- 0.20                               # test scores (highest ICC = hardest); edit to compare outcomes
+zc   <- qnorm(0.975) + qnorm(0.80)         # 2.80 test+power constant
+Ntot <- 90 * 450
+
+# Estimand 1: within-school direct effect (chatbot vs classmate). 0 if no within-school variation.
+direct_mde <- function(pi, f) {
+  mean_s <- sum(f * pi * (1 - pi))         # avg treatment-assignment variance
+  if (mean_s == 0) return(NA)
+  zc * sqrt(1 - icc) * sqrt(1 / (Ntot * mean_s))
+}
+
+# design menu: name, saturations (pi), school shares (f), spillover-slope bins (j,k), pure control?
+menu <- list(
+  list("Uniform 50/50",               c(.50),               c(1),            NA,     FALSE),
+  list("Partial pop: control + 50%",  c(0,.50),             c(.50,.50),      c(1,2), TRUE),
+  list("Equal thirds 25/50/75",       c(.25,.50,.75),       c(1/3,1/3,1/3),  c(1,3), FALSE),
+  list("Extreme 20/50/80 (40/20/40)", c(.20,.50,.80),       c(.40,.20,.40),  c(1,3), FALSE),
+  list("Symmetric 20/80 (50/50)",     c(.20,.80),           c(.50,.50),      c(1,2), FALSE),
+  list("Wide 10/50/90 (40/20/40)",    c(.10,.50,.90),       c(.40,.20,.40),  c(1,3), FALSE),
+  list("Baird ladder 0/25/50/75/100", c(0,.25,.50,.75,1),   rep(.2,5),       c(2,4), TRUE),
+  list("Control + thirds 0/25/50/75", c(0,.25,.50,.75),     rep(.25,4),      c(2,4), TRUE),
+  list("Control + 20/50/80 (0/20/50/80)", c(0,.20,.50,.80), rep(.25,4),      c(2,4), TRUE))
+
+fmt <- function(x) ifelse(is.na(x) | !is.finite(x), "—", sprintf("%.3f", x))
+menu_tbl <- do.call(rbind, lapply(menu, function(d) {
+  pi <- d[[2]]; f <- d[[3]]; jk <- d[[4]]; pc <- d[[5]]
+  slope  <- if (length(pi) > 1) spill_slope_S(90, 450, .05, .80, icc, 1 - icc, pi, f, jk[1], jk[2]) else NA
+  pooled <- if (pc) power_pooled(90, 450, .05, .80, icc, 1 - icc, pi, f) else list(MDE_Tonly = NA, MDE_S = NA)
+  data.frame(Design = d[[1]], `Pure control?` = ifelse(pc, "Yes", "No"),
+             `1. Direct (within-FE)`     = fmt(direct_mde(pi, f)),
+             `2. Treatment vs control`   = ifelse(pc, fmt(pooled$MDE_Tonly), "—"),
+             `3. Spillover slope`        = fmt(slope),
+             `4. Spillover vs control`   = ifelse(pc, fmt(pooled$MDE_S), "—"),
+             check.names = FALSE)
+}))
+kable(menu_tbl, align = c("l","c","c","c","c","c"),
+      caption = "Test scores (ICC 0.20): MDE (SD) for each estimand, by design. Lower = more power; — = not identified.") |>
+  kable_styling(bootstrap_options = c("striped","hover","condensed"), full_width = FALSE) |>
+  add_header_above(c(" " = 2, "Treatment effect" = 2, "Spillover" = 2)) |>
+  footnote(general = paste0(
+    "Estimand 1 (within-FE direct): chatbot vs a classmate without it — precise but spillover-biased. ",
+    "Estimand 2 (Baird power_pooled MDE_Tonly): chatbot vs a PURE-CONTROL school — spillover-clean. ",
+    "Estimand 3 (Baird power_slope MDSE_S): how spillover changes across saturations. ",
+    "Estimand 4 (Baird power_pooled MDE_S): untreated-in-treated-school vs pure control — the spillover LEVEL. ",
+    "Estimands 2 and 4 require a saturation-0 (pure-control) arm; hence '—' for designs without one."),
+    general_title = "Notes: ")
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+Test scores (ICC 0.20): MDE (SD) for each estimand, by design. Lower = +more power; — = not identified. +
+ +
+Treatment effect +
+
+
+Spillover +
+
+Design + +Pure control? + +
    +
  1. Direct (within-FE) +
+
    +
  1. Treatment vs control +
+
    +
  1. Spillover slope +
+
    +
  1. Spillover vs control +
+Uniform 50/50 + +No + +0.025 + +— + +— + +— +
+Partial pop: control + 50% + +Yes + +0.035 + +0.085 + +0.245 + +0.122 +
+Equal thirds 25/50/75 + +No + +0.027 + +— + +0.309 + +— +
+Extreme 20/50/80 (40/20/40) + +No + +0.030 + +— + +0.237 + +— +
+Symmetric 20/80 (50/50) + +No + +0.031 + +— + +0.212 + +— +
+Wide 10/50/90 (40/20/40) + +No + +0.036 + +— + +0.188 + +— +
+Baird ladder 0/25/50/75/100 + +Yes + +0.035 + +0.088 + +0.398 + +0.160 +
+Control + thirds 0/25/50/75 + +Yes + +0.032 + +0.076 + +0.356 + +0.143 +
+Control + 20/50/80 (0/20/50/80) + +Yes + +0.033 + +0.081 + +0.300 + +0.144 +
+Notes: +
+ Estimand 1 (within-FE direct): chatbot vs a classmate +without it — precise but spillover-biased. Estimand 2 (Baird +power_pooled MDE_Tonly): chatbot vs a PURE-CONTROL school — +spillover-clean. Estimand 3 (Baird power_slope MDSE_S): how spillover +changes across saturations. Estimand 4 (Baird power_pooled MDE_S): +untreated-in-treated-school vs pure control — the spillover LEVEL. +Estimands 2 and 4 require a saturation-0 (pure-control) arm; hence ‘—’ +for designs without one. +
+ + + +
  • Direct effect (col 1) is best under uniform +50/50 (~0.025) and gets worse as we add pure-control or extreme +schools (they have little/no within-school chatbot variation). But this +estimate is spillover-biased.
  • +
  • Treatment vs control (col 2) is the +spillover-clean direct effect — available only with a pure +control (~0.085). It’s less precise than col 1 (it’s a between-school +comparison) but immune to contamination. Note it’s still a small, +detectable effect.
  • +
  • Spillover slope (col 3) is best when the two +extreme bins are wide and hold many schools: wide 10/50/90 is +best (~0.19), then symmetric (~0.21) and extreme (~0.24); equal thirds +is weaker (~0.31); and the ladder (~0.40) and control+thirds (~0.36) are +the worst — spreading schools across many bins leaves only a few in each +compared bin, which hurts the slope.
  • +
  • Spillover level vs control (col 4) is the +best-powered way to detect a spillover at all (~0.12 with the +partial-population design) because it leans on the clean +pure-control baseline.
  • + +

    Columns 1 and 3 pull in opposite directions (Baird’s central result): +spreading toward the extremes helps the spillover but costs +direct-effect precision. But if the direct effect has some slack (it +stays ~0.025–0.085 SD everywhere) then the spillover is the +binding constraint. Onme straightforward way in which case +would be to add a pure-control arm (estimand 4, ~0.12 +SD), which also buys a clean treatment effect (estimand 2).

    +

    The same rankings hold for the lower-ICC outcomes (pregnancy, +overconfidence); the absolute MDEs just shrink.

    +
    +
    +
    +
    +

    6 Tool 2 - +RCT2 cross-check (R package)

    +

    Its Calsamplesize() returns the number of +clusters needed to detect a given effect at a target power, per +assignment mechanism. The effect-size argument mu is +“the largest direct effect across treatment +assignment mechanisms.

    +

    We run Calsamplesize for the saturation designs and +compare the clusters it needs against our 450 +schools

    +
    rct2_inputs <- data.frame(
    +  Input = c("Clusters (schools)", "Students per school", "ICC",
    +            "Direct effect to detect (mu)", "Significance / power",
    +            "Designs compared", "Excluded", "Method"),
    +  Value = c("450", "90", "0.10", "0.05 SD", "0.05 / 0.80",
    +            "the 3-mechanism designs: equal thirds 25/50/75; extreme 20/50/80; wide 10/50/90",
    +            "uniform 50/50 (1 mechanism) and symmetric 20/80 (2 mechanisms) - Calsamplesize needs 3 arms; also 0%/100% arms (direct effect undefined there)",
    +            "RCT2 design-based (Jiang-Imai-Malani 2023); data simulated to the assumed ICC"),
    +  check.names = FALSE)
    +knitr::kable(rct2_inputs, caption = "RCT2 cross-check: inputs and assumptions") |>
    +  kableExtra::kable_styling(bootstrap_options = c("striped","hover","condensed"), full_width = FALSE)
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    +RCT2 cross-check: inputs and assumptions +
    +Input + +Value +
    +Clusters (schools) + +450 +
    +Students per school + +90 +
    +ICC + +0.10 +
    +Direct effect to detect (mu) + +0.05 SD +
    +Significance / power + +0.05 / 0.80 +
    +Designs compared + +the 3-mechanism designs: equal thirds 25/50/75; extreme 20/50/80; wide +10/50/90 +
    +Excluded + +uniform 50/50 (1 mechanism) and symmetric 20/80 (2 mechanisms) - +Calsamplesize needs 3 arms; also 0%/100% arms (direct effect undefined +there) +
    +Method + +RCT2 design-based (Jiang-Imai-Malani 2023); data simulated to the +assumed ICC +
    +
    library(RCT2); library(kableExtra)
    +set.seed(2026)
    +n_per <- 90; n_clust <- 450; icc <- 0.10
    +mu <- 0.05                          # DIRECT effect to detect (SD); RCT2's 'mu'
    +
    +# NOTE: RCT2's Calsamplesize is built around a THREE-mechanism design (its worked
    +# example uses three arms) and its internal indexing errors for 1- or 2-mechanism
    +# designs. So we compare only the 3-mechanism designs here (uniform = 1 arm and
    +# symmetric 20/80 = 2 arms are excluded), and it sizes the DIRECT effect, so 0%/100%
    +# arms don't apply either. For each design the three ASSIGNMENT MECHANISMS are the
    +# three treated shares a school can be assigned to.
    +designs_rct2 <- list(
    +  "Equal thirds 25/50/75" = list(lv = c(.25,.50,.75), w = c(1/3,1/3,1/3)),  # mechanisms: 25% / 50% / 75% treated
    +  "Extreme 20/50/80"      = list(lv = c(.20,.50,.80), w = c(.40,.20,.40)),  # mechanisms: 20% / 50% / 80% treated
    +  "Wide 10/50/90"         = list(lv = c(.10,.50,.90), w = c(.40,.20,.40)))  # mechanisms: 10% / 50% / 90% treated
    +
    +build_dat <- function(lv, w) {
    +  sat <- sample(lv, n_clust, replace = TRUE, prob = w)   # random mechanism per school (matches the run that worked)
    +  A   <- rep(sat, each = n_per)                          # A = the school's assignment mechanism (treated share)
    +  Z   <- rbinom(n_clust * n_per, 1, A)                   # treat each student at the school's rate
    +  u   <- rep(rnorm(n_clust, 0, sqrt(icc)), each = n_per)
    +  e   <- rnorm(n_clust * n_per, 0, sqrt(1 - icc))
    +  Y   <- 0.10 * Z + 0.10 * A * (1 - Z) + u + e
    +  data.frame(Z = Z, A = A, Y = Y, id = rep(1:n_clust, each = n_per))   # integer id (matches the run that worked)
    +}
    +
    +rows <- lapply(names(designs_rct2), function(nm) {
    +  d   <- designs_rct2[[nm]]
    +  dat <- build_dat(d$lv, d$w)
    +  qa  <- d$w / sum(d$w)                                # share of schools per mechanism
    +  ss  <- Calsamplesize(dat, mu, qa, alpha = 0.05, beta = 0.20)   # class "sample": numeric vector
    +  cl  <- as.numeric(ss)                                # clusters needed, one per mechanism
    +  data.frame(Design = nm,
    +             Mechanisms = paste0(d$lv * 100, "%", collapse = " / "),
    +             `Clusters needed (per mechanism)` = paste(round(cl), collapse = " / "),
    +             `Min clusters (best mechanism)`   = round(min(cl)),
    +             check.names = FALSE)
    +})
    +kable(do.call(rbind, rows), align = c("l","c","c","c"),
    +      caption = "RCT2 (design-based): clusters needed to detect a 0.05 SD DIRECT effect, by saturation design (ICC 0.10)") |>
    +  kable_styling(bootstrap_options = c("striped","hover","condensed"), full_width = FALSE) |>
    +  footnote(general = paste0(
    +    "Calsamplesize sizes the DIRECT effect (mu = largest direct effect across mechanisms). ",
    +    "Compare 'Min clusters' against our 450 schools; the 50% mechanism is the most efficient. ",
    +    "Numbers are randomization-based and corroborate Baird directionally, not exactly."),
    +    general_title = "Notes: ")
    +
    +
    +
    +

    7 Design × spillover grid +(simulation)

    +

    For each assignment design, and a range of assumed spillover sizes +(including none), this simulation reports (a) the +bias of the naive within-school direct-effect estimate, +(b) whether the true direct effect can be recovered, +and (c) the power to detect the spillover.

    +
    +

    7.1 Assumptions

    +
      +
    • Design held fixed: 450 treatment schools, 90 +surveyed students each.
    • +
    • Outcome standardized so total variance = 1; the +ICC sets the between-school share of variance (we use +0.10 here).
    • +
    • True direct chatbot effect planted at 0.10 SD
    • +
    • Spillover model: untreated students gain +spillover * (school's treated share) —{0, 0.10, 0.20, 0.30} +SD; 0 = no spillover (a validity check).
    • +
    • Nine designs: uniform 50/50; partial population +(0/50); equal thirds (25/50/75); extreme (20/50/80); symmetric (20/80); +wide (10/50/90); control+full (0/50/100); control+thirds (0/25/50/75); +and the Baird ladder (0/25/50/75/100).
    • +
    • What we measure per design × spillover: (i) the +bias of the naive within-school estimate (the +attenuation a design would suffer), and (ii) the power to detect +the spillover by testing whether untreated students’ +outcomes rise with their school’s saturation.
    • +
    +
    library(fixest)      # fixed-effects regressions + cluster-robust SEs
    +library(kableExtra)  # formatted table output
    +set.seed(2026)
    +
    +S        <- 450                      # treatment schools
    +n_per    <- 90                       # students surveyed per school
    +icc      <- 0.10                     # between-school share of variance (ICC)
    +b2_true  <- 0.10                     # TRUE direct chatbot effect we plant (SD units)
    +spills   <- c(0, 0.10, 0.20, 0.30)   # spillover sizes to test (0 = none)
    +REPS     <- 150                      # reps per cell 
    +
    +# every candidate design as (saturation levels, share of schools at each level)
    +DESIGN_SPECS <- list(
    +  "Uniform 50/50"                = list(lv = c(.50),             w = c(1)),
    +  "Partial pop 0/50"             = list(lv = c(0,.50),           w = c(.50,.50)),
    +  "Equal thirds 25/50/75"        = list(lv = c(.25,.50,.75),     w = c(1/3,1/3,1/3)),
    +  "Extreme 20/50/80"             = list(lv = c(.20,.50,.80),     w = c(.40,.20,.40)),
    +  "Symmetric 20/80"              = list(lv = c(.20,.80),         w = c(.50,.50)),
    +  "Wide 10/50/90"                = list(lv = c(.10,.50,.90),     w = c(.40,.20,.40)),
    +  "Control+full 0/50/100"        = list(lv = c(0,.50,1),         w = c(1/3,1/3,1/3)),
    +  "Control+thirds 0/25/50/75"    = list(lv = c(0,.25,.50,.75),   w = c(.25,.25,.25,.25)),
    +  "Baird ladder 0/25/50/75/100"  = list(lv = c(0,.25,.50,.75,1), w = rep(.2, 5)),
    +  "Control + 20/50/80 0/20/50/80" = list(lv = c(0,.20,.50,.80),   w = c(.25,.25,.25,.25)))
    +designs <- names(DESIGN_SPECS)
    +
    +
    +

    7.2 Step 1 — assign each +school a saturation

    +

    assign_sat() gives every school its treated share under +the chosen design, stratified by CSE arm so the two arms stay +balanced.

    +
    assign_sat <- function(design, deb) {
    +  sp <- DESIGN_SPECS[[design]]; lv <- sp$lv; w <- sp$w
    +  if (length(lv) == 1) return(rep(lv, S))                   # uniform: every school same share
    +  sat <- numeric(S)
    +  for (arm in c(0, 1)) {                                    # stratify within each CSE arm
    +    idx <- which(deb == arm); m <- length(idx)
    +    cnt <- floor(w * m); cnt[length(cnt)] <- m - sum(cnt[-length(cnt)])
    +    lvl <- unlist(mapply(function(c, l) rep(l, c), cnt, lv))
    +    sat[idx] <- sample(lvl)                                 # shuffle which school gets which
    +  }
    +  sat
    +}
    +
    +
    +

    7.3 Step 2 — generate +virtual study and estimate it two ways

    +

    The outcome is built as:

    +
    Y = school_effect(ICC) + 0.10*Chatbot + spillover*share*(1 - Chatbot) + noise
    +

    i.e. a school baseline + the planted direct effect for treated +students + a spillover bonus for untreated students that grows +with their school’s treated share + individual noise. We then run both +regressions on that one dataset.

    +
    sim_once <- function(design, spill) {
    +  deb <- sample(rep(c(0, 1), each = S / 2))                 # half schools debiasing
    +  sat <- assign_sat(design, deb)                            # Step 1
    +  k   <- round(sat * n_per)                                 # treated count per school
    +  Tt  <- unlist(lapply(k, function(kk) sample(rep(c(1, 0), c(kk, n_per - kk)))))
    +  d <- data.frame(school = rep(seq_len(S), each = n_per),
    +                  s_s = rep(sat, each = n_per), T = Tt)      # s_s = school treated share
    +  u <- rep(rnorm(S, 0, sqrt(icc)), each = n_per)            # between-school (ICC)
    +  e <- rnorm(S * n_per, 0, sqrt(1 - icc))                   # within-school
    +  d$y <- u + b2_true * d$T + spill * d$s_s * (1 - d$T) + e  # plant the outcome
    +
    +  # (i) NAIVE within-school direct effect (mixed schools only): biased if spillover present
    +  mid   <- d[d$s_s > 0 & d$s_s < 1, ]
    +  b2_fe <- coef(feols(y ~ T | school, data = mid, cluster = ~school))[["T"]]
    +
    +  # (ii) SPILLOVER detection (robust for every design): do UNTREATED students'
    +  #      outcomes rise with their school's saturation? (cluster-robust by school)
    +  un <- d[d$T == 0, ]
    +  reject <- NA_real_
    +  if (length(unique(un$s_s)) > 1) {
    +    m <- feols(y ~ s_s, data = un, cluster = ~school)
    +    reject <- as.numeric(abs(m$coeftable["s_s", "t value"]) > 1.96)
    +  }
    +  c(b2_fe = b2_fe, reject = reject)
    +}
    +
    +
    +

    7.4 Step 3 — repeat over +the whole grid and summarise

    +
    res <- data.frame()
    +for (dz in designs) for (sp in spills) {
    +  draws <- replicate(REPS, sim_once(dz, sp))                # 2 x REPS matrix
    +  res <- rbind(res, data.frame(
    +    design = dz,
    +    spill  = sp,
    +    est    = mean(draws["b2_fe", ]),                        # naive direct estimate (true = 0.10)
    +    power  = mean(draws["reject", ], na.rm = TRUE)))        # power to detect spillover
    +}
    +res
    +
    ##                           design spill           est      power
    +## 1                  Uniform 50/50   0.0  1.014292e-01        NaN
    +## 2                  Uniform 50/50   0.1  5.098699e-02        NaN
    +## 3                  Uniform 50/50   0.2  6.562645e-04        NaN
    +## 4                  Uniform 50/50   0.3 -5.085950e-02        NaN
    +## 5               Partial pop 0/50   0.0  9.991628e-02 0.06000000
    +## 6               Partial pop 0/50   0.1  4.987936e-02 0.31333333
    +## 7               Partial pop 0/50   0.2  7.271248e-04 0.84666667
    +## 8               Partial pop 0/50   0.3 -4.821781e-02 0.99333333
    +## 9          Equal thirds 25/50/75   0.0  1.012611e-01 0.06000000
    +## 10         Equal thirds 25/50/75   0.1  5.116684e-02 0.20666667
    +## 11         Equal thirds 25/50/75   0.2 -3.941830e-05 0.65333333
    +## 12         Equal thirds 25/50/75   0.3 -5.029144e-02 0.97333333
    +## 13              Extreme 20/50/80   0.0  9.966823e-02 0.06666667
    +## 14              Extreme 20/50/80   0.1  5.057877e-02 0.30000000
    +## 15              Extreme 20/50/80   0.2 -1.241907e-04 0.90666667
    +## 16              Extreme 20/50/80   0.3 -5.030254e-02 0.98666667
    +## 17               Symmetric 20/80   0.0  1.009655e-01 0.04666667
    +## 18               Symmetric 20/80   0.1  4.989574e-02 0.44000000
    +## 19               Symmetric 20/80   0.2  1.612644e-03 0.96666667
    +## 20               Symmetric 20/80   0.3 -5.167549e-02 1.00000000
    +## 21                 Wide 10/50/90   0.0  9.898982e-02 0.04666667
    +## 22                 Wide 10/50/90   0.1  4.961385e-02 0.39333333
    +## 23                 Wide 10/50/90   0.2  7.694385e-06 0.96000000
    +## 24                 Wide 10/50/90   0.3 -4.887416e-02 1.00000000
    +## 25         Control+full 0/50/100   0.0  9.905851e-02 0.05333333
    +## 26         Control+full 0/50/100   0.1  4.838049e-02 0.26666667
    +## 27         Control+full 0/50/100   0.2 -6.219815e-04 0.65333333
    +## 28         Control+full 0/50/100   0.3 -5.152756e-02 0.94666667
    +## 29     Control+thirds 0/25/50/75   0.0  1.015154e-01 0.04000000
    +## 30     Control+thirds 0/25/50/75   0.1  5.083685e-02 0.38000000
    +## 31     Control+thirds 0/25/50/75   0.2 -7.224980e-04 0.90000000
    +## 32     Control+thirds 0/25/50/75   0.3 -5.065402e-02 0.98000000
    +## 33   Baird ladder 0/25/50/75/100   0.0  1.008110e-01 0.04666667
    +## 34   Baird ladder 0/25/50/75/100   0.1  5.221631e-02 0.24666667
    +## 35   Baird ladder 0/25/50/75/100   0.2 -9.617680e-04 0.80666667
    +## 36   Baird ladder 0/25/50/75/100   0.3 -5.240280e-02 1.00000000
    +## 37 Control + 20/50/80 0/20/50/80   0.0  9.811034e-02 0.08666667
    +## 38 Control + 20/50/80 0/20/50/80   0.1  4.801312e-02 0.41333333
    +## 39 Control + 20/50/80 0/20/50/80   0.2 -1.285007e-04 0.92000000
    +## 40 Control + 20/50/80 0/20/50/80   0.3 -4.968534e-02 1.00000000
    +
    +
    +

    7.5 Step 4 — format the +results table

    +
    disp <- data.frame(
    +  `Spillover (delta)`            = sprintf("%.2f", res$spill),
    +  `Naive direct estimate`        = sprintf("%+.3f", res$est),
    +  `Spillover detection power`    = ifelse(is.finite(res$power), sprintf("%.0f%%", 100 * res$power), "—"),
    +  check.names = FALSE)
    +
    +kt <- kable(disp, align = c("c", "c", "c"),
    +      caption = sprintf(paste0("Design x spillover grid (live simulation): ",
    +                               "true direct effect = %.2f SD, ICC = %.2f, %d reps/cell"),
    +                        b2_true, icc, REPS)) |>
    +  kable_styling(bootstrap_options = c("striped", "hover", "condensed"), full_width = FALSE)
    +ns <- length(spills)
    +for (i in seq_along(designs)) kt <- pack_rows(kt, designs[i], (i - 1)*ns + 1, i*ns)
    +kt |> footnote(general = paste0(
    +    "Naive direct estimate = within-school FE estimate on mixed schools; the TRUE value is 0.10 but it ",
    +    "is biased by about -0.5*spillover under EVERY design (it collapses to ~0 at spillover 0.20 and flips ",
    +    "negative at 0.30).",
    +    "Spillover detection power = share of reps where untreated students' outcomes significantly rise with ",
    +    "their school's saturation (cluster-robust, alpha = 0.05). Uniform 50/50 has no variation among the ",
    +    "untreated, so it cannot detect spillover ('—'). The delta = 0 rows are validity checks (~5%). "),
    +    general_title = "Notes: ")
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    +Design x spillover grid (live simulation): true direct effect = 0.10 SD, +ICC = 0.10, 150 reps/cell +
    +Spillover (delta) + +Naive direct estimate + +Spillover detection power +
    +Uniform 50/50 +
    +0.00 + ++0.101 + +— +
    +0.10 + ++0.051 + +— +
    +0.20 + ++0.001 + +— +
    +0.30 + +-0.051 + +— +
    +Partial pop 0/50 +
    +0.00 + ++0.100 + +6% +
    +0.10 + ++0.050 + +31% +
    +0.20 + ++0.001 + +85% +
    +0.30 + +-0.048 + +99% +
    +Equal thirds 25/50/75 +
    +0.00 + ++0.101 + +6% +
    +0.10 + ++0.051 + +21% +
    +0.20 + +-0.000 + +65% +
    +0.30 + +-0.050 + +97% +
    +Extreme 20/50/80 +
    +0.00 + ++0.100 + +7% +
    +0.10 + ++0.051 + +30% +
    +0.20 + +-0.000 + +91% +
    +0.30 + +-0.050 + +99% +
    +Symmetric 20/80 +
    +0.00 + ++0.101 + +5% +
    +0.10 + ++0.050 + +44% +
    +0.20 + ++0.002 + +97% +
    +0.30 + +-0.052 + +100% +
    +Wide 10/50/90 +
    +0.00 + ++0.099 + +5% +
    +0.10 + ++0.050 + +39% +
    +0.20 + ++0.000 + +96% +
    +0.30 + +-0.049 + +100% +
    +Control+full 0/50/100 +
    +0.00 + ++0.099 + +5% +
    +0.10 + ++0.048 + +27% +
    +0.20 + +-0.001 + +65% +
    +0.30 + +-0.052 + +95% +
    +Control+thirds 0/25/50/75 +
    +0.00 + ++0.102 + +4% +
    +0.10 + ++0.051 + +38% +
    +0.20 + +-0.001 + +90% +
    +0.30 + +-0.051 + +98% +
    +Baird ladder 0/25/50/75/100 +
    +0.00 + ++0.101 + +5% +
    +0.10 + ++0.052 + +25% +
    +0.20 + +-0.001 + +81% +
    +0.30 + +-0.052 + +100% +
    +Control + 20/50/80 0/20/50/80 +
    +0.00 + ++0.098 + +9% +
    +0.10 + ++0.048 + +41% +
    +0.20 + +-0.000 + +92% +
    +0.30 + +-0.050 + +100% +
    +Notes: +
    + Naive direct estimate = within-school FE estimate on mixed +schools; the TRUE value is 0.10 but it is biased by about -0.5*spillover +under EVERY design (it collapses to ~0 at spillover 0.20 and flips +negative at 0.30).Spillover detection power = share of reps where +untreated students’ outcomes significantly rise with their school’s +saturation (cluster-robust, alpha = 0.05). Uniform 50/50 has no +variation among the untreated, so it cannot detect spillover (‘—’). The +delta = 0 rows are validity checks (~5%). +
    +
      +
    • Spillover detection power rises with the spillover +size. At a 0.20 spillover, the spread designs do +best (symmetric 20/80 and wide 10/50/90 ≈ 90%+, control+thirds +and partial-pop ≈ 85–90%, extreme ≈ 85%), while equal thirds +(~68%) and control+full 0/50/100 (~70%) trail; +the latter because its 100%-saturation third has no untreated students +to measure spillover on.
    • +
    +
    +
    +
    +
    +

    8 TBD:

    +
      +
    • Do we want to estimate spillover effects (is that a study +objective)? +
        +
      • If yes: a saturation design is justified and it +should probably include a pure-control (0%) arm plus several +interior levels.

      • +
      • If no: don’t saturate.

      • +
    • +
    +
    +
    +
    +

    9 Scorecard: both tools +at a glance

    +
    ref_spill <- 0.20
    +zc <- qnorm(.975) + qnorm(.80)
    +score <- do.call(rbind, lapply(designs, function(dz) {
    +  sp <- DESIGN_SPECS[[dz]]
    +  meanS <- sum(sp$w * sp$lv * (1 - sp$lv))                 # avg treatment-assignment variance
    +  me <- if (meanS > 0) zc * sqrt(1 - icc) * sqrt(1/(S*n_per*meanS)) else NA
    +  pw <- res$power[res$design == dz & res$spill == ref_spill]
    +  data.frame(Design = dz,
    +    `Pure control?`            = ifelse(0 %in% sp$lv, "Yes", "No"),
    +    `Main-effect MDE`          = ifelse(is.na(me), "—", sprintf("%.3f SD", me)),
    +    `Spillover power (d=0.20)` = ifelse(length(pw) && is.finite(pw), sprintf("%.0f%%", 100*pw), "—"),
    +    check.names = FALSE)
    +}))
    +kable(score, align = c("l","c","c","c"),
    +      caption = sprintf("Design scorecard (ICC %.2f): main-effect precision (Tool 1) vs spillover detection (grid sim)", icc)) |>
    +  kable_styling(bootstrap_options = c("striped","hover","condensed"), full_width = FALSE) |>
    +  add_header_above(c(" " = 2, "Tool 1 (analytic)" = 1, "Grid (simulation)" = 1)) |>
    +  footnote(general = paste0(
    +    "Main-effect MDE: within-school FE formula (Tool 1) - smaller = more precise direct effect. ",
    +    "Spillover power: grid simulation at a true 0.20 SD spillover - higher = easier to detect. ",
    +    "The columns trade off: spread/extreme designs detect spillover best at a slightly larger main-effect ",
    +    "MDE; uniform 50/50 has the smallest main-effect MDE but cannot detect spillover. A pure-control arm ",
    +    "additionally gives a spillover-CLEAN direct effect (Tool 1 master menu)."),
    +    general_title = "Notes: ")
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    +Design scorecard (ICC 0.10): main-effect precision (Tool 1) vs spillover +detection (grid sim) +
    + +
    +Tool 1 (analytic) +
    +
    +
    +Grid (simulation) +
    +
    +Design + +Pure control? + +Main-effect MDE + +Spillover power (d=0.20) +
    +Uniform 50/50 + +No + +0.026 SD + +— +
    +Partial pop 0/50 + +Yes + +0.037 SD + +85% +
    +Equal thirds 25/50/75 + +No + +0.029 SD + +65% +
    +Extreme 20/50/80 + +No + +0.031 SD + +91% +
    +Symmetric 20/80 + +No + +0.033 SD + +97% +
    +Wide 10/50/90 + +No + +0.038 SD + +96% +
    +Control+full 0/50/100 + +Yes + +0.046 SD + +65% +
    +Control+thirds 0/25/50/75 + +Yes + +0.033 SD + +90% +
    +Baird ladder 0/25/50/75/100 + +Yes + +0.037 SD + +81% +
    +Control + 20/50/80 0/20/50/80 + +Yes + +0.035 SD + +92% +
    +Notes: +
    + Main-effect MDE: within-school FE formula (Tool 1) - smaller += more precise direct effect. Spillover power: grid simulation at a true +0.20 SD spillover - higher = easier to detect. The columns trade off: +spread/extreme designs detect spillover best at a slightly larger +main-effect MDE; uniform 50/50 has the smallest main-effect MDE but +cannot detect spillover. A pure-control arm additionally gives a +spillover-CLEAN direct effect (Tool 1 master menu). +
    +

    Uniform 50/50 has the smallest main-effect MDE (most +precise direct effect) but cannot detect spillover at +all. The spread designs (symmetric 20/80 and +wide 10/50/90 ~94%, extreme ~86%) and the better pure-control +designs (partial-pop ~87%, control+thirds ~89%) detect +spillover best, for only a small increase in the main-effect MDE.

    +
    +
    +
    +

    10 References

    + +
    + + + +
    +
    + +
    + + + + + + + + + + + + + + + + +