Skip to content

Implement fix for mojibake #37

Description

@audiomuze
import polars as pl
import polars.selectors as cs

def repair_all_mojibake(df: pl.DataFrame) -> pl.DataFrame:
    """
    Vectorized cleanup for every string column in a Polars DataFrame.
    Fixes Greek, Cyrillic, Central European, and global UTF-8 encoding corruptions.
    """
    
    # Target all columns containing text strings
    string_cols = cs.string()
    
    return df.with_columns(
        pl.coalesce(
            # 1. Try treating as Global UTF-8 Mojibake (e.g., Café -> Café)
            pl.col(string_cols).str.encode("cp1252").str.decode("utf-8", strict=False),
            
            # 2. Try treating as Legacy Greek Mojibake (e.g., Ñüæá -> Ρόζα)
            pl.col(string_cols).str.encode("cp1252").str.decode("cp1253", strict=False),
            
            # 3. Try treating as Legacy Cyrillic Mojibake (e.g., Çåìôèñà -> Земфира)
            pl.col(string_cols).str.encode("cp1252").str.decode("cp1251", strict=False),
            
            # 4. Try treating as Legacy Central European (e.g., Krakã³w -> Kraków)
            pl.col(string_cols).str.encode("cp1252").str.decode("cp1250", strict=False),
            
            # 5. Fallback: If no decoding strategy worked, keep the original raw text untouched
            pl.col(string_cols)
        ).alias(cs.expand()) # Keeps original column names intact
    )

# =====================================================================
# Demonstration & Verification
# =====================================================================

# Simulated messy multi-column music database
dirty_df = pl.DataFrame({
    "song_title": ["Ñüæá", "Café Blue", "Çåìôèñà Track", "Clean English Song"],
    "artist_name": ["Íôáëüñáò", "Niño Pã©rez", "Unknown", "Stamatis Kokotas"],
    "album_id": [101, 102, 103, 104] # Integers are automatically skipped by the selector
})

clean_df = repair_all_mojibake(dirty_df)
print(clean_df)

Activity

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment

Metadata

Metadata

Assignees

No one assigned

    Labels

    No labels
    No labels

    Projects

    No projects

      Milestone

      No milestone

      Relationships

      None yet

      Development

      No branches or pull requests

      Issue actions