data_cleaner.py

import pandas as pd
import numpy as np
import warnings
warnings.filterwarnings('ignore')

def load_and_clean(path='data/games.csv'):
    print("Loading dataset...")
    df = pd.read_csv(path)
    print(f"Raw shape: {df.shape}")

    # ── Rename columns for easier use ──
    df.columns = [c.strip().lower().replace(' ', '_') for c in df.columns]

    # ── Keep only useful columns ──
    cols = [
        'name', 'release', 'price', 'discount',
        'positive', 'negative', 'user_score',
        'average', 'median', 'developers', 'publishers',
        'categories', 'genres', 'tags',
        'windows', 'mac', 'linux',
        'metacritic_score', 'achievements',
        'supported_languages'
    ]
    # Only keep cols that exist
    cols = [c for c in cols if c in df.columns]
    df = df[cols].copy()

    print(f"Columns kept: {cols}")

    # ── Release date → datetime ──
    if 'release' in df.columns:
        df['release_date'] = pd.to_datetime(df['release'], errors='coerce', format='mixed')
        df['release_year'] = df['release_date'].dt.year
        df['release_month'] = df['release_date'].dt.month

    # ── Price cleaning ──
    if 'price' in df.columns:
        df['price'] = pd.to_numeric(df['price'], errors='coerce')
        df['is_free'] = df['price'] == 0
        df['price_category'] = pd.cut(
            df['price'].fillna(0),
            bins=[-1, 0, 5, 10, 20, 30, 60, 1000],
            labels=['Free', 'Under $5', '$5-$10', '$10-$20', '$20-$30', '$30-$60', '$60+']
        )

    # ── Review metrics ──
    if 'positive' in df.columns and 'negative' in df.columns:
        df['positive'] = pd.to_numeric(df['positive'], errors='coerce').fillna(0)
        df['negative'] = pd.to_numeric(df['negative'], errors='coerce').fillna(0)
        df['total_reviews'] = df['positive'] + df['negative']
        df['positive_rate'] = np.where(
            df['total_reviews'] > 0,
            df['positive'] / df['total_reviews'] * 100,
            np.nan
        )
        df['review_category'] = pd.cut(
            df['positive_rate'].fillna(0),
            bins=[-1, 40, 60, 70, 80, 95, 100],
            labels=['Overwhelmingly Negative', 'Mostly Negative',
                    'Mixed', 'Mostly Positive', 'Very Positive',
                    'Overwhelmingly Positive']
        )

    # ── Numeric columns ──
    for col in ['user_score', 'average', 'median', 'metacritic_score', 'achievements']:
        if col in df.columns:
            df[col] = pd.to_numeric(df[col], errors='coerce')

    # ── Genre extraction (first genre only) ──
    if 'genres' in df.columns:
        df['primary_genre'] = df['genres'].apply(
    lambda x: str(x).split(',')[0].strip() if pd.notna(x) and str(x) != 'nan' else 'Unknown'
        )

    # ── Language count ──
    if 'supported_languages' in df.columns:
        df['language_count'] = df['supported_languages'].astype(str).apply(
            lambda x: len(x.split(',')) if x != 'nan' else 0
        )

    # ── Drop rows with no name ──
    if 'name' in df.columns:
        df = df.dropna(subset=['name'])

    # ── Filter to games with at least some data ──
    if 'total_reviews' in df.columns:
        df_reviewed = df[df['total_reviews'] >= 10].copy()
    else:
        df_reviewed = df.copy()

    print(f"\nCleaned shape (all): {df.shape}")
    print(f"Cleaned shape (reviewed games): {df_reviewed.shape}")
    print(f"\nMissing values:\n{df.isnull().sum()[df.isnull().sum() > 0]}")

    return df, df_reviewed

if __name__ == '__main__':
    df, df_r = load_and_clean()
    print("\nSample:")
    print(df_r[['name', 'price', 'positive_rate', 'primary_genre']].head(10))
    df.to_csv('data/games_cleaned.csv', index=False)
    df_r.to_csv('data/games_reviewed.csv', index=False)
    print("\nSaved cleaned files!")