cycle pooling

Author: eswarchandravidyasagar

app.py

@@ -16,7 +16,7 @@
     load_cycle_variable_info, load_crosswalk, load_categories, merge_data
 )
 from src.data.processor import create_age_groups, apply_region_filter, apply_inclusion_flag_filters
-from src.data.harmonizer import prepare_pooled_variable
+from src.data.harmonizer import filter_poolable_variables, prepare_pooled_variable
 from src.analysis.bootstrap import (
     run_bootstrap_analysis_for_all_values,
     run_cycle_pooled_analysis,
@@ -90,6 +90,7 @@ def main():
     analysis_mode = create_analysis_mode_selector()
     is_pooling_mode = analysis_mode == "Cycle Pooling"
     is_multi_cycle_mode = analysis_mode in {"Multi-Cycle Trends", "Cycle Pooling"}
+    pooling_compatibility_issues = {}
 
     # Handle mode switching with warning if user has active work
     previous_mode = get_session_state('analysis_mode')
@@ -203,6 +204,13 @@ def main():
 
         # Load cycle variable info for all selected cycles (for labels)
         cycle_var_info_dict = {cycle: load_cycle_variable_info(cycle) for cycle in selected_cycles}
+        if is_pooling_mode:
+            available_harmonized_vars, pooling_compatibility_issues = filter_poolable_variables(
+                available_harmonized_vars,
+                selected_cycles,
+                crosswalk,
+                cycle_var_info_dict,
+            )
 
         merged_desc_dict = merge_descriptions(json_desc_dict, desc_dict)
 
@@ -215,6 +223,20 @@ def main():
 
         display_data_metrics(data)
         st.info(f"📊 Data harmonized and combined from cycles: {cycles_str}")
+        if is_pooling_mode and pooling_compatibility_issues:
+            st.warning(
+                f"Excluded {len(pooling_compatibility_issues):,} common column(s) "
+                "from pooling because their meanings or category structures "
+                "are not equivalent across all selected cycles."
+            )
+            with st.expander("Review variables excluded from pooling"):
+                compatibility_df = pd.DataFrame(
+                    [
+                        {"Variable": variable, "Reason": reason}
+                        for variable, reason in pooling_compatibility_issues.items()
+                    ]
+                )
+                st.dataframe(compatibility_df, use_container_width=True)
 
         cycle = cycles_str
         use_harmonized = True
@@ -338,7 +360,7 @@ def main():
         # Get available harmonized variables
         if is_multi_cycle_mode:
             # For multi-cycle, use common harmonized variables
-            if not available_harmonized_vars and crosswalk:
+            if not available_harmonized_vars and crosswalk and not is_pooling_mode:
                 from src.data.harmonizer import get_common_harmonized_vars
                 data_dict_for_check = {}
                 for cycle_year in selected_cycles:
@@ -375,7 +397,7 @@ def main():
                 use_harmonized = True
                 st.info(f"Multi-cycle mode: Using {len(available_harmonized_vars)} harmonized variables available across all selected cycles")
 
-            if use_harmonized and available_harmonized_vars:
+            if use_harmonized:
                 variable_options = available_harmonized_vars
                 variable_labels = {}
                 for var in variable_options:

src/data/harmonizer.py

@@ -1,5 +1,8 @@
 """Data harmonization functions for multi-cycle CCHS analysis."""
 
+import re
+import unicodedata
+
 import pandas as pd
 from typing import Optional
 
@@ -15,6 +18,95 @@ def _category_key(value):
     return str(value)
 
 
+def _normalize_metadata_text(value):
+    text = unicodedata.normalize("NFKC", str(value or "")).casefold()
+    return " ".join(re.sub(r"[^\w]+", " ", text).split())
+
+
+def assess_harmonized_compatibility(
+    variable: str,
+    cycles: list,
+    crosswalk: dict,
+    cycle_variable_info: dict,
+):
+    """Return whether a crosswalk entry is safe to pool and the reason.
+
+    Availability is not enough for pooling. This conservative check requires
+    equivalent normalized descriptions and equivalent category-label sets in
+    every selected cycle. Continuous variables are accepted when all cycles
+    consistently have no categorical metadata.
+    """
+    variable_crosswalk = crosswalk.get(variable, {})
+    metadata = []
+    for cycle_value in cycles:
+        cycle = str(cycle_value)
+        cycle_variable = variable_crosswalk.get(cycle)
+        if not cycle_variable:
+            return False, f"No crosswalk variable for cycle {cycle}."
+        variable_info = cycle_variable_info.get(cycle, {}).get(cycle_variable)
+        if not variable_info:
+            return False, f"No data-dictionary metadata for {cycle_variable} in {cycle}."
+        description = _normalize_metadata_text(variable_info.get("description"))
+        if not description:
+            return False, f"No description for {cycle_variable} in cycle {cycle}."
+        category_labels = {
+            _normalize_metadata_text(label)
+            for label in variable_info.get("categories", {}).values()
+            if _normalize_metadata_text(label)
+        }
+        metadata.append((cycle, cycle_variable, description, category_labels))
+
+    descriptions = {item[2] for item in metadata}
+    if len(descriptions) != 1:
+        details = "; ".join(
+            f"{cycle} {cycle_variable}"
+            for cycle, cycle_variable, _, _ in metadata
+        )
+        return False, f"Descriptions differ across mapped variables ({details})."
+
+    category_sets = [item[3] for item in metadata]
+    categorical_cycles = [
+        metadata[index][0]
+        for index, labels in enumerate(category_sets)
+        if labels
+    ]
+    if categorical_cycles and len(categorical_cycles) != len(metadata):
+        missing = [
+            metadata[index][0]
+            for index, labels in enumerate(category_sets)
+            if not labels
+        ]
+        return False, (
+            "Categorical metadata is missing for cycle(s): "
+            + ", ".join(missing)
+            + "."
+        )
+    if category_sets and any(labels != category_sets[0] for labels in category_sets[1:]):
+        return False, "Response category labels differ across cycles."
+
+    return True, "Descriptions and category structures match across cycles."
+
+
+def filter_poolable_variables(
+    variables: list,
+    cycles: list,
+    crosswalk: dict,
+    cycle_variable_info: dict,
+):
+    """Split common columns into poolable variables and incompatibility reasons."""
+    poolable = []
+    issues = {}
+    for variable in variables:
+        compatible, reason = assess_harmonized_compatibility(
+            variable, cycles, crosswalk, cycle_variable_info
+        )
+        if compatible:
+            poolable.append(variable)
+        else:
+            issues[variable] = reason
+    return poolable, issues
+
+
 def prepare_pooled_variable(
     data: pd.DataFrame,
     variable: str,

tests/test_cycle_pooling.py

@@ -4,7 +4,12 @@
 import pytest
 
 from src.analysis.bootstrap import run_cycle_pooled_analysis
-from src.data.harmonizer import POOLED_VALUE_COLUMN, prepare_pooled_variable
+from src.data.harmonizer import (
+    POOLED_VALUE_COLUMN,
+    assess_harmonized_compatibility,
+    filter_poolable_variables,
+    prepare_pooled_variable,
+)
 
 
 def test_pooling_scales_weights_to_average_annual_population():
@@ -243,3 +248,99 @@ def test_cycle_dictionary_crosswalk_detects_genuinely_missing_categories():
             cycle_variable_info=cycle_info,
             crosswalk=crosswalk,
         )
+
+
+def test_harmonized_compatibility_accepts_matching_meaning_and_categories():
+    crosswalk = {
+        "SMKDVSTY": {"2023": "SMKDVSTY", "2024": "SMKDVSTY"}
+    }
+    variable_info = {
+        cycle: {
+            "SMKDVSTY": {
+                "description": "Smoking status (type 2) - traditional definition - (D)",
+                "categories": {
+                    "01": "Current daily smoker",
+                    "03": "Former daily smoker (non-smoker now)",
+                },
+            }
+        }
+        for cycle in ("2023", "2024")
+    }
+
+    compatible, reason = assess_harmonized_compatibility(
+        "SMKDVSTY", ["2023", "2024"], crosswalk, variable_info
+    )
+
+    assert compatible is True
+    assert "match" in reason
+
+
+def test_harmonized_compatibility_rejects_same_name_with_different_meaning():
+    crosswalk = {"STATUS": {"2023": "STATUS", "2024": "STATUS"}}
+    variable_info = {
+        "2023": {
+            "STATUS": {"description": "Current smoking status", "categories": {}}
+        },
+        "2024": {
+            "STATUS": {"description": "Former smoking status", "categories": {}}
+        },
+    }
+
+    compatible, reason = assess_harmonized_compatibility(
+        "STATUS", ["2023", "2024"], crosswalk, variable_info
+    )
+
+    assert compatible is False
+    assert "Descriptions differ" in reason
+
+
+def test_harmonized_compatibility_rejects_changed_categories():
+    crosswalk = {"STATUS": {"2023": "OLD_STATUS", "2024": "STATUS"}}
+    variable_info = {
+        "2023": {
+            "OLD_STATUS": {
+                "description": "Smoking status",
+                "categories": {"1": "Daily", "2": "Occasional"},
+            }
+        },
+        "2024": {
+            "STATUS": {
+                "description": "Smoking status",
+                "categories": {"1": "Daily", "2": "Never"},
+            }
+        },
+    }
+
+    compatible, reason = assess_harmonized_compatibility(
+        "STATUS", ["2023", "2024"], crosswalk, variable_info
+    )
+
+    assert compatible is False
+    assert "category labels differ" in reason
+
+
+def test_poolable_filter_excludes_false_crosswalk_match():
+    crosswalk = {
+        "SAFE": {"2023": "SAFE", "2024": "SAFE"},
+        "SPU_10": {"2023": "SPU_10", "2024": "SPU_10B"},
+    }
+    variable_info = {
+        "2023": {
+            "SAFE": {"description": "Exact measure", "categories": {}},
+            "SPU_10": {"description": "Stopped smoking - when", "categories": {}},
+        },
+        "2024": {
+            "SAFE": {"description": "Exact measure", "categories": {}},
+            "SPU_10B": {"description": "Stopped smoking - year", "categories": {}},
+        },
+    }
+
+    poolable, issues = filter_poolable_variables(
+        ["SAFE", "SPU_10"],
+        ["2023", "2024"],
+        crosswalk,
+        variable_info,
+    )
+
+    assert poolable == ["SAFE"]
+    assert "SPU_10" in issues