Update helper functions

Author: amaanjaved1

README.md

@@ -116,8 +116,8 @@ from rmp_client import (
 )
 ```
 
-- `normalize_comment(text)` — Normalize text for deduplication (lowercase, collapse whitespace)
-- `is_valid_comment(text, min_len=10)` — Check if a comment is non-empty and meets a minimum length
+- `normalize_comment(text, *, strip_html=True, strip_punctuation=False)` — Normalize text for deduplication (trim, strip HTML, lowercase, collapse whitespace; optionally strip punctuation)
+- `is_valid_comment(text, *, min_len=10)` — Validate a comment and return a `ValidationResult` with diagnostics (empty, too short, all caps, excessive repeats, no alpha)
 - `clean_course_label(raw)` — Clean scraped course labels (remove counts, normalize whitespace)
 - `build_course_mapping(scraped, valid)` — Map scraped labels to known course codes
 - `analyze_sentiment(text)` — Compute sentiment label from text (uses TextBlob)

docs/extras.html

@@ -46,12 +46,42 @@ <h2 id="sentiment">Sentiment <a href="#sentiment" class="anchor">#</a></h2>
     <pre><code class="language-python">result = analyze_sentiment("Great prof, explains concepts clearly.")
 print(result.score, result.label)  # e.g. 0.65 "positive"</code></pre>
 
-    <h2 id="dedupe-helpers">Dedupe Helpers <a href="#dedupe-helpers" class="anchor">#</a></h2>
-    <p>Normalize comments for deduplication and filter out low-quality entries.</p>
-    <pre><code class="language-python">raw = "  This prof is AMAZING!!!  "
-normalized = normalize_comment(raw)  # "this prof is amazing!!!"
-if is_valid_comment(normalized, min_len=10):
-    print("Valid comment")</code></pre>
+    <h2 id="helpers">Helpers <a href="#helpers" class="anchor">#</a></h2>
+
+    <h3 id="normalize_comment">normalize_comment</h3>
+    <div class="method-sig">normalize_comment(text: str, *, strip_html: bool = True, strip_punctuation: bool = False) -&gt; str</div>
+    <p>Normalizes a comment for comparison or deduplication. Trims whitespace, strips HTML tags (opt-out), lowercases, and collapses runs of whitespace. Optionally strips punctuation for looser matching.</p>
+    <table>
+      <tr><th>Parameter</th><th>Type</th><th>Default</th><th>Description</th></tr>
+      <tr><td><code>text</code></td><td><code>str</code></td><td>&mdash;</td><td>Comment text</td></tr>
+      <tr><td><code>strip_html</code></td><td><code>bool</code></td><td><code>True</code></td><td>Remove HTML tags</td></tr>
+      <tr><td><code>strip_punctuation</code></td><td><code>bool</code></td><td><code>False</code></td><td>Remove all punctuation</td></tr>
+    </table>
+    <pre><code class="language-python">a = normalize_comment("  Great  Professor!  ")
+b = normalize_comment("great professor!")
+assert a == b  # True
+
+normalize_comment("&lt;b&gt;Loved&lt;/b&gt; this class")  # "loved this class"
+normalize_comment("Hello, world!", strip_punctuation=True)  # "hello world"</code></pre>
+
+    <h3 id="is_valid_comment">is_valid_comment</h3>
+    <div class="method-sig">is_valid_comment(text: str, *, min_len: int = 10) -&gt; ValidationResult</div>
+    <p>Validates a comment and returns detailed diagnostics. Checks for empty text, insufficient length, all-caps, excessive repeated characters, and absence of alphabetic characters.</p>
+    <table>
+      <tr><th>Parameter</th><th>Type</th><th>Default</th><th>Description</th></tr>
+      <tr><td><code>text</code></td><td><code>str</code></td><td>&mdash;</td><td>Comment text</td></tr>
+      <tr><td><code>min_len</code></td><td><code>int</code></td><td><code>10</code></td><td>Minimum character length</td></tr>
+    </table>
+    <p><strong>Returns:</strong> <code>ValidationResult</code> with <code>valid</code> (bool) and <code>issues</code> (list of <code>CommentIssue</code>).</p>
+    <p>Each issue has a <code>code</code> (<code>"empty"</code>, <code>"too_short"</code>, <code>"all_caps"</code>, <code>"excessive_repeats"</code>, <code>"no_alpha"</code>) and a human-readable <code>message</code>.</p>
+    <pre><code class="language-python">result = is_valid_comment("Good")
+# ValidationResult(valid=False, issues=[CommentIssue(code="too_short", ...)])
+
+result = is_valid_comment("Great class, learned a lot")
+# ValidationResult(valid=True, issues=[])
+
+result = is_valid_comment("WORST PROF EVER!!!")
+# ValidationResult(valid=False, issues=[CommentIssue(code="all_caps", ...)])</code></pre>
 
     <h2 id="course-code-helpers">Course Code Helpers <a href="#course-code-helpers" class="anchor">#</a></h2>
     <p>Map scraped RMP course labels to your course catalog.</p>

docs/index.html

@@ -55,7 +55,7 @@ <h2 id="features">Features <a href="#features" class="anchor">#</a></h2>
       <li>In-memory caching for ratings pages</li>
       <li>Cursor-based pagination for all list/search endpoints</li>
       <li>Clear error hierarchy for precise exception handling</li>
-      <li>Built-in helpers for ingestion workflows (sentiment, dedupe, course codes)</li>
+      <li>Built-in helpers for ingestion workflows (sentiment, comment validation, course codes)</li>
     </ul>
 
     <h2 id="requirements">Requirements <a href="#requirements" class="anchor">#</a></h2>
@@ -82,7 +82,7 @@ <h2 id="documentation">Documentation <a href="#documentation" class="anchor">#</
       <li><a href="usage.html">Usage</a> &mdash; Quickstart examples for every endpoint</li>
       <li><a href="configuration.html">Configuration</a> &mdash; Tuning retries, rate limits, timeouts, and headers</li>
       <li><a href="reference.html">API Reference</a> &mdash; Full method and type reference</li>
-      <li><a href="extras.html">Extras</a> &mdash; Ingestion helpers (sentiment, dedupe, course mapping)</li>
+      <li><a href="extras.html">Extras</a> &mdash; Ingestion helpers (sentiment, comment validation, course mapping)</li>
     </ul>
   </main>
 

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "ratemyprofessors-client"
-version = "2.0.0"
+version = "2.1.0"
 description = "Typed, retrying, rate-limited unofficial Python client for the RateMyProfessors GraphQL API."
 readme = "README.md"
 requires-python = ">=3.10"

src/rmp_client/__init__.py

@@ -15,6 +15,8 @@
 from .extras import (
     SentimentResult,
     analyze_sentiment,
+    CommentIssue,
+    ValidationResult,
     is_valid_comment,
     normalize_comment,
     build_course_mapping,
@@ -34,6 +36,8 @@
     "TokenBucket",
     "SentimentResult",
     "analyze_sentiment",
+    "CommentIssue",
+    "ValidationResult",
     "is_valid_comment",
     "normalize_comment",
     "build_course_mapping",

src/rmp_client/extras/__init__.py

@@ -1,13 +1,15 @@
-# Ingestion helpers: sentiment, dedupe, course_codes.
+# Ingestion helpers: sentiment, helpers, course_codes.
 # Re-exported from rmp_client so you can: from rmp_client import analyze_sentiment, ...
 
 from .sentiment import SentimentResult, analyze_sentiment
-from .dedupe import is_valid_comment, normalize_comment
+from .helpers import CommentIssue, ValidationResult, is_valid_comment, normalize_comment
 from .course_codes import build_course_mapping, clean_course_label
 
 __all__ = [
     "SentimentResult",
     "analyze_sentiment",
+    "CommentIssue",
+    "ValidationResult",
     "is_valid_comment",
     "normalize_comment",
     "build_course_mapping",

src/rmp_client/extras/dedupe.py

@@ -0,0 +1,100 @@
+"""Helpers for normalizing and validating rating comments."""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass, field
+from typing import Literal
+
+
+def _strip_html(text: str) -> str:
+    """Strip HTML tags from text (RMP comments occasionally contain markup)."""
+    return re.sub(r"<[^>]*>", "", text)
+
+
+def normalize_comment(
+    text: str,
+    *,
+    strip_html: bool = True,
+    strip_punctuation: bool = False,
+) -> str:
+    """Normalize a comment for comparison or deduplication.
+
+    - Trims leading/trailing whitespace
+    - Strips HTML tags (opt-out via *strip_html*)
+    - Lowercases
+    - Collapses runs of whitespace to a single space
+    - Optionally strips punctuation for looser matching
+    """
+    out = text.strip()
+    if strip_html:
+        out = _strip_html(out)
+    out = re.sub(r"\s+", " ", out.lower())
+    if strip_punctuation:
+        out = re.sub(r"[^\w\s]", "", out)
+    return out
+
+
+IssueCode = Literal[
+    "empty",
+    "too_short",
+    "all_caps",
+    "excessive_repeats",
+    "no_alpha",
+]
+
+
+@dataclass
+class CommentIssue:
+    code: IssueCode
+    message: str
+
+
+@dataclass
+class ValidationResult:
+    valid: bool
+    issues: list[CommentIssue] = field(default_factory=list)
+
+
+def is_valid_comment(text: str, *, min_len: int = 10) -> ValidationResult:
+    """Validate a comment and return detailed diagnostics.
+
+    Checks for:
+    - Empty or whitespace-only text
+    - Below minimum length (*min_len*, default 10)
+    - All uppercase (shouting)
+    - Excessive repeated characters (e.g. "aaaaaaa")
+    - No alphabetic characters at all
+    """
+    issues: list[CommentIssue] = []
+    trimmed = (text or "").strip()
+
+    if not trimmed:
+        issues.append(CommentIssue(code="empty", message="Comment is empty"))
+        return ValidationResult(valid=False, issues=issues)
+
+    if len(trimmed) < min_len:
+        issues.append(
+            CommentIssue(
+                code="too_short",
+                message=f"Comment is {len(trimmed)} chars (minimum {min_len})",
+            )
+        )
+
+    if len(trimmed) > 3 and trimmed == trimmed.upper() and re.search(r"[A-Z]", trimmed):
+        issues.append(CommentIssue(code="all_caps", message="Comment is all uppercase"))
+
+    if re.search(r"(.)\1{4,}", trimmed, re.IGNORECASE):
+        issues.append(
+            CommentIssue(
+                code="excessive_repeats",
+                message="Comment contains excessive repeated characters",
+            )
+        )
+
+    if not re.search(r"[a-zA-Z]", trimmed):
+        issues.append(
+            CommentIssue(code="no_alpha", message="Comment contains no alphabetic characters")
+        )
+
+    return ValidationResult(valid=len(issues) == 0, issues=issues)