From d896aa67ef01f5659be0cedfd3e3dbd332e99223 Mon Sep 17 00:00:00 2001 From: Alejandro Ortiz Giraldo Date: Thu, 23 Jul 2026 13:30:20 -0500 Subject: [PATCH] Implement HFC checks 1,3-6 per DQAP coverage mapping - 1-import: read .csv (raw is CSV, not DTA); recode -888/-999 sentinels to missing (keep -666 = "Other" gate on hh_watersource/stored_container) - 3-missingness: consent + all always-asked fields + Section D conditional on stored_yn gate, share missing by enumerator (data_02) - 4-outliers: Tukey fences on resp_age, hh_size, hh_children, storage_time (excl. 99), treat_chlorine, treat_boil, duration_min, child_age_* (data_03) - 5-constraints: fixed ranges + storage_time 0-72-or-99 + relational hh_children <= hh_size (data_04) - 6-surveylogic: consent/stored_yn/hh_children skip gates, Other-specify mismatches, diarrhea reverse skip, roster count, head-under-18 (data_05) --- code/1-import.do | 14 ++- code/3-missingness.do | 67 +++++++----- code/4-outliers.do | 96 ++++++++++++------ code/5-constraints.do | 137 ++++++++++++++++++------- code/6-surveylogic.do | 230 +++++++++++++++++++++++++++++++++++------- 5 files changed, 411 insertions(+), 133 deletions(-) diff --git a/code/1-import.do b/code/1-import.do index 06018cc..3bfd271 100644 --- a/code/1-import.do +++ b/code/1-import.do @@ -13,10 +13,10 @@ ... one column set per repeat). *******************************************************************************/ - use "${data}/raw/${data_hwq}.dta", clear + import delimited "${data}/raw/${data_hwq}.csv", clear varnames(1) bindquote(strict) - * If you only have the .csv: - // import delimited "${data}/raw/${data_hwq}.csv", clear varnames(1) bindquote(strict) + * If you have the .dta instead: + // use "${data}/raw/${data_hwq}.dta", clear describe, short count @@ -31,6 +31,14 @@ capture confirm variable duration_min if _rc gen duration_min = duration / 60 + * --- Sentinel codes --------------------------------------------------------- + * -888 (refused) and -999 (don't know) are MISSING for every downstream check. + * Recode them to Stata missing once, here, so `missing()` catches them. + * -666 is DIFFERENT: on hh_watersource (C7) and stored_container (D2) it is + * the valid "Other" answer that gates the _o specify field, so leave it. + ds, has(type numeric) + mvdecode `r(varlist)', mv(-888 -999) + save "${data}/raw/${data_hwq}_use.dta", replace * How many surveys per day? Per enumerator? Get a feel for the data first. diff --git a/code/3-missingness.do b/code/3-missingness.do index b5a29b8..6e3adc1 100644 --- a/code/3-missingness.do +++ b/code/3-missingness.do @@ -1,9 +1,16 @@ /******************************************************************************* - 3-missingness.do · HFC exercise [YOUR TURN] + 3-missingness.do · HFC exercise -------------------------------------------------------------------------------- - DQAP row this answers: Check = Missing values | Variables = consent (done - for you) + the water-storage module, Section D (your turn) | - Who acts = DMS programmer first, field trainer if the code turns out fine. + DQAP row this answers: Check = Missing values | Variables = consent (the + response-rate gate) + every always-asked field + Section D conditional on + its gate | Who acts = DMS programmer first, field trainer if the code is fine. + + Rule (from the DQAP coverage table): + - always-asked fields count as missing on every consenting survey; + - conditional fields count as missing only when their gate fired; + - a refusal (consent == 0) is SUPPOSED to be blank, so exclude it; + - -888 / -999 already recoded to missing in 1-import.do; -666 = "Other" + is a real answer and is NOT missing. Output: sheet 02_Missingness of the HFC excel — one row per enumerator, with the share of their surveys missing each key item. @@ -11,38 +18,42 @@ use "${data}/raw/${data_hwq}_use.dta", clear - * --- COMPLETED: consent should basically never be missing --- - * (all answers are numeric-coded, so "missing" = Stata's . missing) + * --- consent: the response-rate gate, should basically never be missing --- gen byte _miss_consent = missing(consent) tab _miss_consent - * --- YOUR TURN --- - * Your DQAP flagged the water-storage module, Section D (stored_yn, - * stored_covered, stored_clean, storage_time, stored_chlorine). One - * enumerator's surveys are almost entirely blank on this module. Write - * the check that would catch it — look at missingness BY enumerator, - * not just overall. - * - * NOTE: only count surveys with consent == 1 — a refusal is SUPPOSED to - * be blank! (And remember: if stored_yn == 0, the rest of Section D is - * also SUPPOSED to be blank — the gate question itself is the one that - * should never be missing.) - * - * Hint structure (replace ___ ): - * gen byte _miss_storage = missing(___) if consent == ___ - -* --- Export: one row per enumerator (extend this with your new variable!) --- + * --- Always-asked fields: missing among CONSENTING surveys only ----------- + * (A2 hh_id, A3 village_id, A4 gps, A5 starttime/endtime, C1-C7, D1, + * E1-E3, F1-F2, H2 duration_min) + local always /// + hh_id village_id gps starttime endtime /// + resp_age resp_sex resp_hh_head resp_educ hh_size hh_children hh_watersource /// + stored_yn treat_chlorine treat_boil treat_notablets /// + water_safety water_satisfaction duration_min + + foreach v of local always { + gen byte _miss_`v' = missing(`v') if consent == 1 + } + + * --- Section D storage module: conditional on the D1 gate (stored_yn==1) --- + * Blank is correct when stored_yn != 1, so only count those. + local storage_d stored_container stored_covered stored_clean storage_time stored_chlorine + foreach v of local storage_d { + gen byte _miss_`v' = missing(`v') if consent == 1 & stored_yn == 1 + } + +* --- Export: one row per enumerator, share missing on each item --------------- preserve gen byte _has_hhid = !missing(hh_id) keep if consent == 1 - collapse (count) n_surveys=_has_hhid (mean) pct_miss_consent=_miss_consent /// - /* (mean) pct_miss_storage=_miss_storage */ , by(enumerator) - * gsort -pct_miss_storage - format pct_miss_* %9.2f + collapse (count) n_surveys=_has_hhid (mean) _miss_*, by(enumerator) + rename _miss_* pct_miss_* + format pct_miss_* %9.3f export excel using "${hfc_excel}", sheet("data_02") /// firstrow(variables) sheetmodify restore - * >> Once exported, which enumerator jumps out, and is this a programming - * problem (broken form on their device) or a field problem? + * >> Which enumerator jumps out on pct_miss_stored_*? A whole module blank + * for one enumerator = broken form on their device (programming), not a + * field problem. diff --git a/code/4-outliers.do b/code/4-outliers.do index f7344c5..57e3254 100644 --- a/code/4-outliers.do +++ b/code/4-outliers.do @@ -1,42 +1,74 @@ /******************************************************************************* - 4-outliers.do · HFC exercise [YOUR TURN] + 4-outliers.do · HFC exercise -------------------------------------------------------------------------------- - DQAP row this answers: Check = Outliers | Variables = ___ (you decide, - based on what's numeric and bounded in the questionnaire) | - Who acts = enumerator/supervisor callback to confirm the figure. + DQAP row this answers: Check = Outliers | Variables = resp_age, hh_size, + hh_children, storage_time, treat_chlorine, treat_boil, child_age_*, + duration_min | Who acts = enumerator/supervisor callback to confirm. - Candidates from the questionnaire: storage_time, hh_size, child_age_* - (the children roster is WIDE: child_age_1, child_age_2, ...). + Method: Tukey fences. For each variable, flag values below Q1-1.5*IQR or + above Q3+1.5*IQR. This catches values far outside the sample WITHOUT hard- + coding a range (that is the constraints check, file 5). - CAREFUL: storage_time uses 99 as a code for "more than 72 hours" — a 99 - is a VALID answer, not an outlier. Exclude it before flagging! + CAREFUL: storage_time uses 99 = "more than 72 hours". A 99 is a VALID code, + not an outlier — drop it before computing the fences and flagging. Output: sheet 03_Outliers of the HFC excel — one row per flagged value: - hh_id, enumerator, variable, value. That's a callback list. + check, hh_id, enumerator, variable, value. That is a callback list. *******************************************************************************/ use "${data}/raw/${data_hwq}_use.dta", clear - * Start here: look at the distributions before you set any bounds - summarize storage_time hh_size child_age_*, detail - - * --- YOUR TURN --- - * For at least one variable, list the households whose value looks - * physically implausible or far outside the rest of the sample, and - * export them so the supervisor has a callback list. - * - * Hint structure (replace ___ ): - * preserve - * keep if ___ > ___ & ___ != 99 & !missing(___) - * keep hh_id enumerator ___ - * gen variable = "___" - * rename ___ value - * gen check = "Outlier" - * order check hh_id enumerator variable value - * export excel using "${hfc_excel}", sheet("data_03") /// - * firstrow(variables) sheetmodify - * restore - * - * (To flag SEVERAL variables in one sheet, build each flag list the same - * way, `append` them together, then export once — see the answer key. - * For the children roster, loop over child_age_1 to child_age_6.) + * Look at the distributions before setting any bounds + summarize resp_age hh_size hh_children storage_time /// + treat_chlorine treat_boil duration_min child_age_*, detail + + tempfile base all + save `base' + + local ovars resp_age hh_size hh_children storage_time treat_chlorine /// + treat_boil duration_min child_age_1 child_age_2 child_age_3 + + local first 1 + foreach v of local ovars { + use `base', clear + + * storage_time: 99 is a valid code, exclude it from the fence + the flag + if "`v'" == "storage_time" drop if `v' == 99 + + quietly summarize `v', detail + local iqr = r(p75) - r(p25) + local lo = r(p25) - 1.5*`iqr' + local hi = r(p75) + 1.5*`iqr' + + * Missing (.) counts as "greater than" any number in Stata — exclude it + keep if !missing(`v') & (`v' < `lo' | `v' > `hi') + keep hh_id enumerator `v' + rename `v' value + gen variable = "`v'" + + if _N > 0 { + if `first' { + save `all', replace + local first 0 + } + else { + append using `all' + save `all', replace + } + } + } + +* --- Export: one callback row per flagged value ------------------------------ + + if `first' == 0 { + use `all', clear + gen check = "Outlier" + order check hh_id enumerator variable value + gsort variable -value + export excel using "${hfc_excel}", sheet("data_03") /// + firstrow(variables) sheetmodify + } + + * >> Each row is a household to call back, not proof of an error: a hh_size + * of 25 might be real. The list is the conversation, the callback is the + * verdict. diff --git a/code/5-constraints.do b/code/5-constraints.do index 14a8d11..530a788 100644 --- a/code/5-constraints.do +++ b/code/5-constraints.do @@ -1,45 +1,112 @@ /******************************************************************************* - 5-constraints.do · HFC exercise [YOUR TURN] + 5-constraints.do · HFC exercise -------------------------------------------------------------------------------- - DQAP row this answers: Check = Constraint violations | Variables = ___ - (anything with a defined valid range in the questionnaire) | + DQAP row this answers: Check = Constraint violations | Variables = every + field with a defined valid range + the relational hh_children <= hh_size | Who acts = DMS programmer adds a hard constraint to the form. - Reminder of the questionnaire's valid ranges: - resp_age 18-100 - hh_size 1-20 - hh_children 0 - hh_size (relational: cannot exceed hh_size!) - treat_chlorine 0-7 - treat_boil 0-7 - storage_time 0-72, or exactly 99 ("more than 72 hours" code) - child_age_* 0-60 months + Valid ranges (from the questionnaire): + hh_id > 0 + resp_age 18-100 + hh_size 1-20 + hh_children 0 - hh_size (relational: cannot exceed hh_size!) + treat_chlorine 0-7 + treat_boil 0-7 + storage_time 0-72, or exactly 99 ("more than 72 hours" code) + child_age_* 0-60 months + + CAREFUL: missing (.) counts as "greater than" any number in Stata — exclude + it. storage_time's 99 is valid. -888/-999 already recoded missing. Output: sheet 04_Constraints of the HFC excel — one row per violation: - hh_id, enumerator, variable, value, valid_range. + check, hh_id, enumerator, variable, value, valid_range. *******************************************************************************/ use "${data}/raw/${data_hwq}_use.dta", clear - * --- YOUR TURN --- - * Pick at least two of the ranges above, list any rows that violate them, - * and export the violations. (Careful: missing (.) counts as "greater - * than" any number in Stata — exclude it! And remember storage_time's - * valid 99 code.) - * - * Hint structure (replace ___ ): - * preserve - * keep if (___ < ___ | ___ > ___) & !missing(___) - * keep hh_id enumerator ___ - * gen variable = "___" - * rename ___ value - * gen valid_range = "[___, ___]" - * gen check = "Constraint violation" - * order check hh_id enumerator variable value valid_range - * export excel using "${hfc_excel}", sheet("data_04") /// - * firstrow(variables) sheetmodify - * restore - * - * (Same trick as 03: build one flag list per variable, `append`, export - * once. The answer key shows a loop that does all ranges at once — plus - * the RELATIONAL check hh_children <= hh_size, which no single-variable - * range can catch.) + tempfile base all + save `base' + local first 1 + + * --- Fixed single-variable ranges: name:lo:hi ----------------------------- + local specs resp_age:18:100 hh_size:1:20 treat_chlorine:0:7 treat_boil:0:7 /// + child_age_1:0:60 child_age_2:0:60 child_age_3:0:60 hh_id:1:9999999999 + + foreach s of local specs { + tokenize "`s'", parse(":") + local v `1' + local lo `3' + local hi `5' + + use `base', clear + keep if !missing(`v') & (`v' < `lo' | `v' > `hi') + keep hh_id enumerator `v' + rename `v' value + gen variable = "`v'" + gen valid_range = "[`lo', `hi']" + + if _N > 0 { + if `first' { + save `all', replace + local first 0 + } + else { + append using `all' + save `all', replace + } + } + } + + * --- storage_time: 0-72 OR the valid 99 code ----------------------------- + use `base', clear + keep if !missing(storage_time) & storage_time != 99 & /// + (storage_time < 0 | storage_time > 72) + keep hh_id enumerator storage_time + rename storage_time value + gen variable = "storage_time" + gen valid_range = "[0, 72] or 99" + if _N > 0 { + if `first' { + save `all', replace + local first 0 + } + else { + append using `all' + save `all', replace + } + } + + * --- Relational: hh_children must not exceed hh_size ---------------------- + * No single-variable range can catch this one. + use `base', clear + keep if !missing(hh_children) & !missing(hh_size) & /// + (hh_children < 0 | hh_children > hh_size) + keep hh_id enumerator hh_children hh_size + rename hh_children value + gen variable = "hh_children" + gen valid_range = "[0, hh_size=" + string(hh_size) + "]" + keep hh_id enumerator value variable valid_range + if _N > 0 { + if `first' { + save `all', replace + local first 0 + } + else { + append using `all' + save `all', replace + } + } + +* --- Export: one row per violation ------------------------------------------- + + if `first' == 0 { + use `all', clear + gen check = "Constraint violation" + order check hh_id enumerator variable value valid_range + sort variable hh_id + export excel using "${hfc_excel}", sheet("data_04") /// + firstrow(variables) sheetmodify + } + + * >> These are hard errors the form should have blocked at entry. Every row + * here is a constraint the DMS programmer should add before tomorrow. diff --git a/code/6-surveylogic.do b/code/6-surveylogic.do index 85477f5..a294d2d 100644 --- a/code/6-surveylogic.do +++ b/code/6-surveylogic.do @@ -1,42 +1,202 @@ /******************************************************************************* - 6-surveylogic.do · HFC exercise [YOUR TURN] + 6-surveylogic.do · HFC exercise -------------------------------------------------------------------------------- DQAP row this answers: Check = Logical inconsistencies / skip-pattern - violations | Variables = ___ | Who acts = both — supervisor callback today, - programmer fixes the constraint before tomorrow's batch. - - Skip patterns in this questionnaire: - - Section B -> everything: consent == 0 should end the interview — all - later sections must be blank. - - D1 (stored_yn == 0) should skip the rest of Section D: container, - covered, clean, storage_time, chlorine must all be blank when the - household has no stored water. - - G (children roster): diarrhea_7d is only asked when diarrhea_2d == 0 — - so a row with diarrhea_2d == 1 AND diarrhea_7d filled broke the skip. - (The roster is WIDE: check diarrhea_2d_1 vs diarrhea_7d_1, etc.) - - There's also a plain logical inconsistency that doesn't involve a skip - pattern at all: does resp_age line up with resp_hh_head? - - Output: sheet 05_SurveyLogic of the HFC excel — one row per violation, - with an `issue` column saying which rule was broken and a - `detail` column showing the contradictory values. + violations | Who acts = both — supervisor callback today, programmer fixes + the constraint before tomorrow's batch. + + Skip patterns & logic checked here (from the DQAP coverage table): + 1. consent == 0 -> interview ends; later sections must be blank. + 2. stored_yn == 0 -> rest of Section D (container/covered/clean/ + storage_time/chlorine) must be blank. + 3. hh_children == 0 -> Section G roster must be empty. + 4. hh_watersource == -666 (Other) <-> hh_watersource_o filled (C7->C8). + 5. stored_container == -666 (Other) <-> stored_container_o filled (D2->D3). + 6. diarrhea_7d only asked when diarrhea_2d == 0 -> 2d==1 & 7d filled breaks + the reverse skip (WIDE roster: check each child 1..3). + 7. roster count of children must equal hh_children. + 8. plain logic: a household head (resp_hh_head==1) younger than 18. + + Output: sheet 05_SurveyLogic of the HFC excel — one row per violation: + check, hh_id, enumerator, issue, detail. *******************************************************************************/ use "${data}/raw/${data_hwq}_use.dta", clear - * --- YOUR TURN --- - * Write a check for at least one of the inconsistencies above and - * export the flagged rows. - * - * Hint structure (replace ___ ): - * preserve - * keep if ___ == 0 & !missing(___) - * keep hh_id enumerator ___ ___ - * gen issue = "___" - * gen detail = "___=0 but ___=" + string(___) - * gen check = "Survey logic" - * order check hh_id enumerator issue detail - * export excel using "${hfc_excel}", sheet("data_05") /// - * firstrow(variables) sheetmodify - * restore + tempfile base all + save `base' + local first 1 + + * Helper macro tail: append current data to `all'. Requires columns + * check hh_id enumerator issue detail. Run via -include- by copy below. + +* --- 1. consent == 0 but downstream fields filled ---------------------------- + use `base', clear + gen byte _bad = consent == 0 & /// + (!missing(resp_age) | !missing(hh_size) | !missing(hh_watersource)) + keep if _bad + gen issue = "Consent=No but survey continued" + gen detail = "consent=0 but resp_age=" + string(resp_age) + ", hh_size=" + string(hh_size) + keep hh_id enumerator issue detail + if _N > 0 { + if `first' { + save `all', replace + local first 0 + } + else { + append using `all' + save `all', replace + } + } + +* --- 2. stored_yn == 0 but Section D filled ---------------------------------- + use `base', clear + gen byte _bad = stored_yn == 0 & /// + (!missing(stored_container) | !missing(stored_covered) | /// + !missing(stored_clean) | !missing(storage_time) | !missing(stored_chlorine)) + keep if _bad + gen issue = "stored_yn=No but Section D filled" + gen detail = "stored_yn=0 but stored_container=" + string(stored_container) + /// + ", storage_time=" + string(storage_time) + keep hh_id enumerator issue detail + if _N > 0 { + if `first' { + save `all', replace + local first 0 + } + else { + append using `all' + save `all', replace + } + } + +* --- 3. hh_children == 0 but roster filled ----------------------------------- + use `base', clear + gen byte _bad = hh_children == 0 & /// + (!missing(child_name_1) & child_name_1 != "" | !missing(child_age_1)) + keep if _bad + gen issue = "hh_children=0 but roster filled" + gen detail = "hh_children=0 but child_name_1=" + child_name_1 + ", child_age_1=" + string(child_age_1) + keep hh_id enumerator issue detail + if _N > 0 { + if `first' { + save `all', replace + local first 0 + } + else { + append using `all' + save `all', replace + } + } + +* --- 4. hh_watersource Other (-666) <-> hh_watersource_o ----------------------- + use `base', clear + gen byte _bad = (hh_watersource == -666 & (missing(hh_watersource_o) | hh_watersource_o == "")) | /// + (hh_watersource != -666 & !missing(hh_watersource) & hh_watersource_o != "") + keep if _bad + gen issue = "hh_watersource Other/specify mismatch" + gen detail = "hh_watersource=" + string(hh_watersource) + " but hh_watersource_o=" + `"""' + hh_watersource_o + `"""' + keep hh_id enumerator issue detail + if _N > 0 { + if `first' { + save `all', replace + local first 0 + } + else { + append using `all' + save `all', replace + } + } + +* --- 5. stored_container Other (-666) <-> stored_container_o -------------------- + use `base', clear + gen byte _bad = (stored_container == -666 & (missing(stored_container_o) | stored_container_o == "")) | /// + (stored_container != -666 & !missing(stored_container) & stored_container_o != "") + keep if _bad + gen issue = "stored_container Other/specify mismatch" + gen detail = "stored_container=" + string(stored_container) + " but stored_container_o=" + `"""' + stored_container_o + `"""' + keep hh_id enumerator issue detail + if _N > 0 { + if `first' { + save `all', replace + local first 0 + } + else { + append using `all' + save `all', replace + } + } + +* --- 6. diarrhea reverse skip: 2d==1 but 7d filled (each child) --------------- + forvalues i = 1/3 { + use `base', clear + capture confirm variable diarrhea_2d_`i' + if _rc continue + gen byte _bad = diarrhea_2d_`i' == 1 & !missing(diarrhea_7d_`i') + keep if _bad + gen issue = "diarrhea_7d asked when diarrhea_2d=Yes (child `i')" + gen detail = "diarrhea_2d_`i'=1 but diarrhea_7d_`i'=" + string(diarrhea_7d_`i') + keep hh_id enumerator issue detail + if _N > 0 { + if `first' { + save `all', replace + local first 0 + } + else { + append using `all' + save `all', replace + } + } + } + +* --- 7. roster count != hh_children ------------------------------------------ + use `base', clear + egen _nchild = rownonmiss(child_age_1 child_age_2 child_age_3) + gen byte _bad = consent == 1 & !missing(hh_children) & hh_children <= 3 & _nchild != hh_children + keep if _bad + gen issue = "Roster count != hh_children" + gen detail = "hh_children=" + string(hh_children) + " but roster has " + string(_nchild) + " children" + keep hh_id enumerator issue detail + if _N > 0 { + if `first' { + save `all', replace + local first 0 + } + else { + append using `all' + save `all', replace + } + } + +* --- 8. logical: household head younger than 18 ------------------------------ + use `base', clear + gen byte _bad = resp_hh_head == 1 & !missing(resp_age) & resp_age < 18 + keep if _bad + gen issue = "Household head under 18" + gen detail = "resp_hh_head=1 but resp_age=" + string(resp_age) + keep hh_id enumerator issue detail + if _N > 0 { + if `first' { + save `all', replace + local first 0 + } + else { + append using `all' + save `all', replace + } + } + +* --- Export: one row per violation ------------------------------------------- + + if `first' == 0 { + use `all', clear + gen check = "Survey logic" + order check hh_id enumerator issue detail + sort issue hh_id + export excel using "${hfc_excel}", sheet("data_05") /// + firstrow(variables) sheetmodify + } + + * >> Skip-pattern breaks (2, 4, 5, 6) are programming bugs — the form let a + * field through it should have hidden. Logic breaks (1, 3, 7, 8) are + * callbacks first: confirm with the enumerator before assuming the worst.