Implement HFC checks 1,3-6 per DQAP coverage mapping

code/1-import.do

@@ -13,10 +13,10 @@
             ... one column set per repeat).
 *******************************************************************************/
 
-	use "${data}/raw/${data_hwq}.dta", clear
+	import delimited "${data}/raw/${data_hwq}.csv", clear varnames(1) bindquote(strict)
 
-	* If you only have the .csv:
-	// import delimited "${data}/raw/${data_hwq}.csv", clear varnames(1) bindquote(strict)
+	* If you have the .dta instead:
+	// use "${data}/raw/${data_hwq}.dta", clear
 
 	describe, short
 	count
@@ -31,6 +31,14 @@
 	capture confirm variable duration_min
 	if _rc gen duration_min = duration / 60
 
+	* --- Sentinel codes ---------------------------------------------------------
+	* -888 (refused) and -999 (don't know) are MISSING for every downstream check.
+	* Recode them to Stata missing once, here, so `missing()` catches them.
+	* -666 is DIFFERENT: on hh_watersource (C7) and stored_container (D2) it is
+	* the valid "Other" answer that gates the _o specify field, so leave it.
+	ds, has(type numeric)
+	mvdecode `r(varlist)', mv(-888 -999)
+
 	save "${data}/raw/${data_hwq}_use.dta", replace
 
 	* How many surveys per day? Per enumerator? Get a feel for the data first.

code/3-missingness.do

@@ -1,48 +1,59 @@
 /*******************************************************************************
-  3-missingness.do  ·  HFC exercise                                 [YOUR TURN]
+  3-missingness.do  ·  HFC exercise
 --------------------------------------------------------------------------------
-  DQAP row this answers:  Check = Missing values | Variables = consent (done
-  for you) + the water-storage module, Section D (your turn) |
-  Who acts = DMS programmer first, field trainer if the code turns out fine.
+  DQAP row this answers:  Check = Missing values | Variables = consent (the
+  response-rate gate) + every always-asked field + Section D conditional on
+  its gate | Who acts = DMS programmer first, field trainer if the code is fine.
+
+  Rule (from the DQAP coverage table):
+    - always-asked fields count as missing on every consenting survey;
+    - conditional fields count as missing only when their gate fired;
+    - a refusal (consent == 0) is SUPPOSED to be blank, so exclude it;
+    - -888 / -999 already recoded to missing in 1-import.do; -666 = "Other"
+      is a real answer and is NOT missing.
 
   Output:  sheet 02_Missingness of the HFC excel — one row per enumerator,
            with the share of their surveys missing each key item.
 *******************************************************************************/
 
 	use "${data}/raw/${data_hwq}_use.dta", clear
 
-	* --- COMPLETED: consent should basically never be missing ---
-	* (all answers are numeric-coded, so "missing" = Stata's . missing)
+	* --- consent: the response-rate gate, should basically never be missing ---
 	gen byte _miss_consent = missing(consent)
 	tab _miss_consent
 
-	* --- YOUR TURN ---
-	* Your DQAP flagged the water-storage module, Section D (stored_yn,
-	* stored_covered, stored_clean, storage_time, stored_chlorine). One
-	* enumerator's surveys are almost entirely blank on this module. Write
-	* the check that would catch it — look at missingness BY enumerator,
-	* not just overall.
-	*
-	* NOTE: only count surveys with consent == 1 — a refusal is SUPPOSED to
-	* be blank! (And remember: if stored_yn == 0, the rest of Section D is
-	* also SUPPOSED to be blank — the gate question itself is the one that
-	* should never be missing.)
-	*
-	* Hint structure (replace ___ ):
-	*   gen byte _miss_storage = missing(___) if consent == ___
-
-* --- Export: one row per enumerator (extend this with your new variable!) ---
+	* --- Always-asked fields: missing among CONSENTING surveys only -----------
+	*     (A2 hh_id, A3 village_id, A4 gps, A5 starttime/endtime, C1-C7, D1,
+	*      E1-E3, F1-F2, H2 duration_min)
+	local always ///
+		hh_id village_id gps starttime endtime ///
+		resp_age resp_sex resp_hh_head resp_educ hh_size hh_children hh_watersource ///
+		stored_yn treat_chlorine treat_boil treat_notablets ///
+		water_safety water_satisfaction duration_min
+
+	foreach v of local always {
+		gen byte _miss_`v' = missing(`v') if consent == 1
+	}
+
+	* --- Section D storage module: conditional on the D1 gate (stored_yn==1) ---
+	*     Blank is correct when stored_yn != 1, so only count those.
+	local storage_d stored_container stored_covered stored_clean storage_time stored_chlorine
+	foreach v of local storage_d {
+		gen byte _miss_`v' = missing(`v') if consent == 1 & stored_yn == 1
+	}
+
+* --- Export: one row per enumerator, share missing on each item ---------------
 
 	preserve
 		gen byte _has_hhid = !missing(hh_id)
 		keep if consent == 1
-		collapse (count) n_surveys=_has_hhid (mean) pct_miss_consent=_miss_consent ///
-			/* (mean) pct_miss_storage=_miss_storage */ , by(enumerator)
-		* gsort -pct_miss_storage
-		format pct_miss_* %9.2f
+		collapse (count) n_surveys=_has_hhid (mean) _miss_*, by(enumerator)
+		rename _miss_* pct_miss_*
+		format pct_miss_* %9.3f
 		export excel using "${hfc_excel}", sheet("data_02") ///
 			firstrow(variables) sheetmodify
 	restore
 
-	* >> Once exported, which enumerator jumps out, and is this a programming
-	*    problem (broken form on their device) or a field problem?
+	* >> Which enumerator jumps out on pct_miss_stored_*? A whole module blank
+	*    for one enumerator = broken form on their device (programming), not a
+	*    field problem.

code/4-outliers.do

@@ -1,42 +1,74 @@
 /*******************************************************************************
-  4-outliers.do  ·  HFC exercise                                    [YOUR TURN]
+  4-outliers.do  ·  HFC exercise
 --------------------------------------------------------------------------------
-  DQAP row this answers:  Check = Outliers | Variables = ___ (you decide,
-  based on what's numeric and bounded in the questionnaire) |
-  Who acts = enumerator/supervisor callback to confirm the figure.
+  DQAP row this answers:  Check = Outliers | Variables = resp_age, hh_size,
+  hh_children, storage_time, treat_chlorine, treat_boil, child_age_*,
+  duration_min | Who acts = enumerator/supervisor callback to confirm.
 
-  Candidates from the questionnaire: storage_time, hh_size, child_age_*
-  (the children roster is WIDE: child_age_1, child_age_2, ...).
+  Method: Tukey fences. For each variable, flag values below Q1-1.5*IQR or
+  above Q3+1.5*IQR. This catches values far outside the sample WITHOUT hard-
+  coding a range (that is the constraints check, file 5).
 
-  CAREFUL:  storage_time uses 99 as a code for "more than 72 hours" — a 99
-            is a VALID answer, not an outlier. Exclude it before flagging!
+  CAREFUL:  storage_time uses 99 = "more than 72 hours". A 99 is a VALID code,
+            not an outlier — drop it before computing the fences and flagging.
 
   Output:  sheet 03_Outliers of the HFC excel — one row per flagged value:
-           hh_id, enumerator, variable, value. That's a callback list.
+           check, hh_id, enumerator, variable, value. That is a callback list.
 *******************************************************************************/
 
 	use "${data}/raw/${data_hwq}_use.dta", clear
 
-	* Start here: look at the distributions before you set any bounds
-	summarize storage_time hh_size child_age_*, detail
-
-	* --- YOUR TURN ---
-	* For at least one variable, list the households whose value looks
-	* physically implausible or far outside the rest of the sample, and
-	* export them so the supervisor has a callback list.
-	*
-	* Hint structure (replace ___ ):
-	*   preserve
-	*       keep if ___ > ___ & ___ != 99 & !missing(___)
-	*       keep hh_id enumerator ___
-	*       gen variable = "___"
-	*       rename ___ value
-	*       gen check = "Outlier"
-	*       order check hh_id enumerator variable value
-	*       export excel using "${hfc_excel}", sheet("data_03") ///
-	*           firstrow(variables) sheetmodify
-	*   restore
-	*
-	* (To flag SEVERAL variables in one sheet, build each flag list the same
-	*  way, `append` them together, then export once — see the answer key.
-	*  For the children roster, loop over child_age_1 to child_age_6.)
+	* Look at the distributions before setting any bounds
+	summarize resp_age hh_size hh_children storage_time ///
+		treat_chlorine treat_boil duration_min child_age_*, detail
+
+	tempfile base all
+	save `base'
+
+	local ovars resp_age hh_size hh_children storage_time treat_chlorine ///
+		treat_boil duration_min child_age_1 child_age_2 child_age_3
+
+	local first 1
+	foreach v of local ovars {
+		use `base', clear
+
+		* storage_time: 99 is a valid code, exclude it from the fence + the flag
+		if "`v'" == "storage_time" drop if `v' == 99
+
+		quietly summarize `v', detail
+		local iqr = r(p75) - r(p25)
+		local lo  = r(p25) - 1.5*`iqr'
+		local hi  = r(p75) + 1.5*`iqr'
+
+		* Missing (.) counts as "greater than" any number in Stata — exclude it
+		keep if !missing(`v') & (`v' < `lo' | `v' > `hi')
+		keep hh_id enumerator `v'
+		rename `v' value
+		gen variable = "`v'"
+
+		if _N > 0 {
+			if `first' {
+				save `all', replace
+				local first 0
+			}
+			else {
+				append using `all'
+				save `all', replace
+			}
+		}
+	}
+
+* --- Export: one callback row per flagged value ------------------------------
+
+	if `first' == 0 {
+		use `all', clear
+		gen check = "Outlier"
+		order check hh_id enumerator variable value
+		gsort variable -value
+		export excel using "${hfc_excel}", sheet("data_03") ///
+			firstrow(variables) sheetmodify
+	}
+
+	* >> Each row is a household to call back, not proof of an error: a hh_size
+	*    of 25 might be real. The list is the conversation, the callback is the
+	*    verdict.

code/5-constraints.do

@@ -1,45 +1,112 @@
 /*******************************************************************************
-  5-constraints.do  ·  HFC exercise                                 [YOUR TURN]
+  5-constraints.do  ·  HFC exercise
 --------------------------------------------------------------------------------
-  DQAP row this answers:  Check = Constraint violations | Variables = ___
-  (anything with a defined valid range in the questionnaire) |
+  DQAP row this answers:  Check = Constraint violations | Variables = every
+  field with a defined valid range + the relational hh_children <= hh_size |
   Who acts = DMS programmer adds a hard constraint to the form.
 
-  Reminder of the questionnaire's valid ranges:
-    resp_age          18-100
-    hh_size            1-20
-    hh_children        0 - hh_size   (relational: cannot exceed hh_size!)
-    treat_chlorine     0-7
-    treat_boil         0-7
-    storage_time       0-72, or exactly 99 ("more than 72 hours" code)
-    child_age_*        0-60 months
+  Valid ranges (from the questionnaire):
+    hh_id          > 0
+    resp_age       18-100
+    hh_size         1-20
+    hh_children     0 - hh_size   (relational: cannot exceed hh_size!)
+    treat_chlorine  0-7
+    treat_boil      0-7
+    storage_time    0-72, or exactly 99 ("more than 72 hours" code)
+    child_age_*     0-60 months
+
+  CAREFUL:  missing (.) counts as "greater than" any number in Stata — exclude
+            it. storage_time's 99 is valid. -888/-999 already recoded missing.
 
   Output:  sheet 04_Constraints of the HFC excel — one row per violation:
-           hh_id, enumerator, variable, value, valid_range.
+           check, hh_id, enumerator, variable, value, valid_range.
 *******************************************************************************/
 
 	use "${data}/raw/${data_hwq}_use.dta", clear
 
-	* --- YOUR TURN ---
-	* Pick at least two of the ranges above, list any rows that violate them,
-	* and export the violations. (Careful: missing (.) counts as "greater
-	* than" any number in Stata — exclude it! And remember storage_time's
-	* valid 99 code.)
-	*
-	* Hint structure (replace ___ ):
-	*   preserve
-	*       keep if (___ < ___ | ___ > ___) & !missing(___)
-	*       keep hh_id enumerator ___
-	*       gen variable = "___"
-	*       rename ___ value
-	*       gen valid_range = "[___, ___]"
-	*       gen check = "Constraint violation"
-	*       order check hh_id enumerator variable value valid_range
-	*       export excel using "${hfc_excel}", sheet("data_04") ///
-	*           firstrow(variables) sheetmodify
-	*   restore
-	*
-	* (Same trick as 03: build one flag list per variable, `append`, export
-	*  once. The answer key shows a loop that does all ranges at once — plus
-	*  the RELATIONAL check hh_children <= hh_size, which no single-variable
-	*  range can catch.)
+	tempfile base all
+	save `base'
+	local first 1
+
+	* --- Fixed single-variable ranges: name:lo:hi -----------------------------
+	local specs resp_age:18:100 hh_size:1:20 treat_chlorine:0:7 treat_boil:0:7 ///
+		child_age_1:0:60 child_age_2:0:60 child_age_3:0:60 hh_id:1:9999999999
+
+	foreach s of local specs {
+		tokenize "`s'", parse(":")
+		local v `1'
+		local lo `3'
+		local hi `5'
+
+		use `base', clear
+		keep if !missing(`v') & (`v' < `lo' | `v' > `hi')
+		keep hh_id enumerator `v'
+		rename `v' value
+		gen variable    = "`v'"
+		gen valid_range = "[`lo', `hi']"
+
+		if _N > 0 {
+			if `first' {
+				save `all', replace
+				local first 0
+			}
+			else {
+				append using `all'
+				save `all', replace
+			}
+		}
+	}
+
+	* --- storage_time: 0-72 OR the valid 99 code -----------------------------
+	use `base', clear
+	keep if !missing(storage_time) & storage_time != 99 & ///
+		(storage_time < 0 | storage_time > 72)
+	keep hh_id enumerator storage_time
+	rename storage_time value
+	gen variable    = "storage_time"
+	gen valid_range = "[0, 72] or 99"
+	if _N > 0 {
+		if `first' {
+			save `all', replace
+			local first 0
+		}
+		else {
+			append using `all'
+			save `all', replace
+		}
+	}
+
+	* --- Relational: hh_children must not exceed hh_size ----------------------
+	*     No single-variable range can catch this one.
+	use `base', clear
+	keep if !missing(hh_children) & !missing(hh_size) & ///
+		(hh_children < 0 | hh_children > hh_size)
+	keep hh_id enumerator hh_children hh_size
+	rename hh_children value
+	gen variable    = "hh_children"
+	gen valid_range = "[0, hh_size=" + string(hh_size) + "]"
+	keep hh_id enumerator value variable valid_range
+	if _N > 0 {
+		if `first' {
+			save `all', replace
+			local first 0
+		}
+		else {
+			append using `all'
+			save `all', replace
+		}
+	}
+
+* --- Export: one row per violation -------------------------------------------
+
+	if `first' == 0 {
+		use `all', clear
+		gen check = "Constraint violation"
+		order check hh_id enumerator variable value valid_range
+		sort variable hh_id
+		export excel using "${hfc_excel}", sheet("data_04") ///
+			firstrow(variables) sheetmodify
+	}
+
+	* >> These are hard errors the form should have blocked at entry. Every row
+	*    here is a constraint the DMS programmer should add before tomorrow.