diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 3f865bb..876899c 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -13,6 +13,9 @@ jobs: steps: - uses: actions/checkout@v7 + - name: Engagement guard + run: python3 scripts/engagement_guard.py + - uses: actions/setup-python@v6 with: python-version: "3.12" diff --git a/.gitignore b/.gitignore index 6e114d8..7375ea5 100644 --- a/.gitignore +++ b/.gitignore @@ -16,3 +16,11 @@ credentials.* .env.* *.pem secrets.* + +# --- lailara engagement scaffold --- +# Client engagement data is runtime-only: never commit it, never deploy it. +client-data/ +client-output/ +/engagement.yml +/engagement.yaml +# (engagement.demo.yml and engagement.example.yml stay committable) diff --git a/CHANGELOG.md b/CHANGELOG.md index d607c3f..98ccb5a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,24 @@ Format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). ## [Unreleased] +### Fixed +- README "What It Detects" now states **9** date patterns (was 6), matching + `DATE_PATTERNS` in `detection.py` (7 phone / 6 currency were already correct). + +### Changed +- Regenerated the committed `samples/output/` reports from current source so the + shipped showcase artifacts reflect the 1.3.0 tool. + +## [1.3.0] - 2026-08-05 + +### Added +- `AuditResult.severity_counts` — a High/Medium/Low breakdown that counts **every** + issue type (findings, duplicates, fuzzy duplicates, schema violations), so it + reconciles to `total_issues`. The findings-only `high_issues` / `medium_issues` + / `low_issues` properties undercounted, because duplicates and schema violations + carry a severity but are not findings — a severity breakdown drawn from them did + not foot to the headline total. `severity_counts` is the reconciling view. + ## [1.2.1] - 2026-07-28 ### Fixed diff --git a/README.md b/README.md index fe0e91e..b3d326b 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ A single run produces three reports tailored to three audiences: an **HTML repor ## What It Detects -**Mixed Formats** — Identifies dates, phone numbers, and currency values stored in inconsistent formats within the same column. For example, `2023-01-15` alongside `Jan 15, 2023` and `01/15/2023` in one date field. The auditor recognizes 6 date patterns, 7 phone patterns, and 6 currency patterns. +**Mixed Formats** — Identifies dates, phone numbers, and currency values stored in inconsistent formats within the same column. For example, `2023-01-15` alongside `Jan 15, 2023` and `01/15/2023` in one date field. The auditor recognizes 9 date patterns, 7 phone patterns, and 6 currency patterns. **Misused Fields** — Flags data stored in the wrong column: reference codes in name fields, free text in currency columns, invalid email addresses, and mixed boolean representations (`Y/N` vs `1/0` vs `Active/Inactive` in the same field). diff --git a/data_hygiene_auditor/api.py b/data_hygiene_auditor/api.py index ae62ec9..c6bcd7d 100644 --- a/data_hygiene_auditor/api.py +++ b/data_hygiene_auditor/api.py @@ -192,6 +192,29 @@ def medium_issues(self) -> List[Finding]: def low_issues(self) -> List[Finding]: return [f for f in self.findings if f.is_low] + @property + def severity_counts(self) -> Dict[str, int]: + """Issue counts by severity across EVERY issue type — findings, + duplicates, fuzzy duplicates, and schema violations — so the breakdown + foots to ``total_issues``. + + ``high_issues`` / ``medium_issues`` / ``low_issues`` are the *findings* + view (Finding objects only); duplicates and schema violations carry a + severity too, so a High/Medium/Low breakdown drawn only from findings + undercounts and does not reconcile to the headline total. This property + is the reconciling breakdown: ``sum(severity_counts.values()) == + total_issues``. + """ + counts = {"High": 0, "Medium": 0, "Low": 0} + for s in self.sheets: + for collection in (s.findings, s.duplicates, s.fuzzy_duplicates, + s.schema_violations): + for issue in collection: + sev = getattr(issue, "severity", None) + if sev in counts: + counts[sev] += 1 + return counts + def to_dict(self) -> Dict[str, Any]: """Return the raw audit results dict.""" return self._raw diff --git a/data_hygiene_auditor/reporting/html.py b/data_hygiene_auditor/reporting/html.py index 7e0845b..c514854 100644 --- a/data_hygiene_auditor/reporting/html.py +++ b/data_hygiene_auditor/reporting/html.py @@ -87,6 +87,15 @@ def _render_fix(fix: dict[str, str]) -> str: ) +# Static one-liner lifted out of the score-hero f-string so no source line exceeds +# the 120-char lint limit; implicit concatenation reproduces the exact same markup. +_SCORE_SCALE_HTML = ( + '
Health score, 0–100 — ' + '90+ clean · 70–89 needs attention · ' + '40–69 significant issues · below 40 critical
' +) + + def generate_html(results: dict[str, Any], output_path: str) -> str: """Generate a client-readable HTML report.""" counts = count_issues(results) @@ -532,7 +541,7 @@ def generate_html(results: dict[str, Any], output_path: str) -> str:
{label}
{score_desc}
-
Health score, 0–100 — 90+ clean · 70–89 needs attention · 40–69 significant issues · below 40 critical
+ {_SCORE_SCALE_HTML}
diff --git a/pyproject.toml b/pyproject.toml index 23c8251..09c2df4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "data-hygiene-auditor" -version = "1.2.1" +version = "1.3.0" description = "A linter for your data — detect mixed formats, misused fields, placeholder floods, and phantom duplicates in Excel and CSV files" readme = "README.md" license = {text = "MIT"} diff --git a/samples/output/sample_messy_data_audit_findings.xlsx b/samples/output/sample_messy_data_audit_findings.xlsx index fdcec2a..0eadb87 100644 Binary files a/samples/output/sample_messy_data_audit_findings.xlsx and b/samples/output/sample_messy_data_audit_findings.xlsx differ diff --git a/samples/output/sample_messy_data_audit_report.html b/samples/output/sample_messy_data_audit_report.html index 6b42b91..a6ab618 100644 --- a/samples/output/sample_messy_data_audit_report.html +++ b/samples/output/sample_messy_data_audit_report.html @@ -217,6 +217,11 @@ flex-shrink: 0; } .score-ring svg { display: block; transform: rotate(-90deg); } +.score-scale { + font-size: 12px; + color: var(--text-secondary, #595959); + margin-top: 6px; +} .score-ring .score-value { position: absolute; top: 50%; @@ -414,7 +419,7 @@

Data Hygiene Audit Report

-

sample_messy_data.xlsx — 2026-07-27 16:58:34

+

sample_messy_data.xlsx — 2026-08-05 20:45:07

@@ -431,6 +436,7 @@

Data Hygiene Audit Report

Significant Issues
This dataset has serious quality problems.
+
Health score, 0–100 — 90+ clean · 70–89 needs attention · 40–69 significant issues · below 40 critical
@@ -489,7 +495,7 @@

Sheet: Customers lambda x: "coded" if isinstance(x, str) and "-" in x else "numeric" )

-
+
FirstName name @@ -522,7 +528,7 @@

Sheet: Customers )

Medium Suspicious repetition — "Doe" appears 3 times (11.5%)
Why this matters: When the same value appears far more often than expected, it may indicate a default value that was never updated, a copy-paste error, or a system glitch that stamped the same data across multiple records.
Suggested Fix (flag_repetitions)
Flag 3 rows where "LastName" = "Doe" (11.5%) for manual review
df["_LastName_review"] = (
     df["LastName"] == "Doe"
 )
-
+
Email email @@ -542,7 +548,7 @@

Sheet: Customers )

Medium Suspicious repetition — "test@test.com" appears 3 times (11.5%)
Why this matters: When the same value appears far more often than expected, it may indicate a default value that was never updated, a copy-paste error, or a system glitch that stamped the same data across multiple records.
Suggested Fix (flag_repetitions)
Flag 3 rows where "Email" = "test@test.com" (11.5%) for manual review
df["_Email_review"] = (
     df["Email"] == "test@test.com"
 )
-
+
Phone phone @@ -566,7 +572,7 @@

Sheet: Customers )

Medium Suspicious repetition — "555-555-5555" appears 3 times (11.5%)
Why this matters: When the same value appears far more often than expected, it may indicate a default value that was never updated, a copy-paste error, or a system glitch that stamped the same data across multiple records.
Suggested Fix (flag_repetitions)
Flag 3 rows where "Phone" = "555-555-5555" (11.5%) for manual review
df["_Phone_review"] = (
     df["Phone"] == "555-555-5555"
 )
-
+
JoinDate date @@ -585,7 +591,7 @@

Sheet: Customers )

Medium Suspicious repetition — "2023-01-15" appears 3 times (11.5%)
Why this matters: When the same value appears far more often than expected, it may indicate a default value that was never updated, a copy-paste error, or a system glitch that stamped the same data across multiple records.
Suggested Fix (flag_repetitions)
Flag 3 rows where "JoinDate" = "2023-01-15" (11.5%) for manual review
df["_JoinDate_review"] = (
     df["JoinDate"] == "2023-01-15"
 )
-
+
AccountBalance currency @@ -609,7 +615,7 @@

Sheet: Customers )

Medium Suspicious repetition — "$1,250.00" appears 3 times (11.5%)
Why this matters: When the same value appears far more often than expected, it may indicate a default value that was never updated, a copy-paste error, or a system glitch that stamped the same data across multiple records.
Suggested Fix (flag_repetitions)
Flag 3 rows where "AccountBalance" = "$1,250.00" (11.5%) for manual review
df["_AccountBalance_review"] = (
     df["AccountBalance"] == "$1,250.00"
 )
-
+
Status categorical @@ -766,7 +772,7 @@

Sheet: Orders df["Amount"].str.replace(r"[^\d.]", "", regex=True) .astype(float) )

-
+
ShipDate date @@ -780,7 +786,7 @@

Sheet: Orders
6 distinct  |  75.0% unique  |  avg len 9.9
Low High missing rate — 2 of 10 values missing (20.0%)
Why this matters: High rates of missing data reduce the reliability of any analysis built on this field. Missing values can skew averages, break joins between tables, and cause downstream systems to error out or produce incomplete results.
Suggested Fix (fill_missing)
Fill 2 missing values in "ShipDate" (20.0%)
df["ShipDate"] = df["ShipDate"].fillna(df["ShipDate"].mode()[0])
High Mixed date formats — 3 of 8 values deviate from YYYY-MM-DD
FormatCount
YYYY-MM-DD5
MM/DD/YYYY1
Mon DD, YYYY1
M/D/YYYY1
Why this matters: Mixed date formats cause sorting failures, broken filters, and incorrect calculations. A date stored as text ("Jan 15, 2023") won't sort chronologically next to "2023-01-15". Downstream tools, APIs, and reports will misparse or reject inconsistent dates.
Suggested Fix (normalize_dates)
Standardize all dates in "ShipDate" to YYYY-MM-DD format
df["ShipDate"] = pd.to_datetime(
     df["ShipDate"], format="mixed", dayfirst=False
 ).dt.strftime("%Y-%m-%d")

-
+
Status categorical @@ -812,7 +818,7 @@

Sheet: Orders OrderIDCustomerIDOrderDateAmountShipDateStatus ORD-006CUST-0102023-01-01$0.002023-01-01TestORD-007CUST-0102023-01-01$0.002023-01-01Test
Why this matters: Exact duplicate rows are the clearest sign of a data quality issue — they can result from double-submissions, ETL failures, or missing unique constraints. Every duplicate inflates counts and distorts any metric built on this data.
Suggested Fix (drop_exact_duplicates)
Remove 2 exact duplicate rows (rows 7, 8)
df = df.drop_duplicates(keep="first").reset_index(drop=True)

- Data Hygiene Audit — Generated 2026-07-27 16:58:34 — Lailara LLC + Data Hygiene Audit — Generated 2026-08-05 20:45:07 — Lailara LLC