data_preprocessing.py

import pandas as pd
import numpy as np
from sklearn.preprocessing import LabelEncoder, MinMaxScaler
import os

def preprocess_books_data(filepath="datasets/booksdata.csv"):
    """
    Preprocess the books dataset.
    - Removes duplicate entries
    - Handles missing values
    - Normalizes ratings from 1-5 to 0-1 scale
    Returns: cleaned pandas DataFrame
    """
    df = pd.read_csv(filepath)
    print(f"Original shape: {df.shape}")

    df = df.drop_duplicates()
    print(f"After removing duplicates: {df.shape}")

    df = df.dropna(subset=['title', 'authors'])
    df['description'] = df['description'].fillna('No description available')

    scaler = MinMaxScaler()
    df['rating_normalized'] = scaler.fit_transform(df[['rating']])

    print(f"Final shape: {df.shape}")
    return df

def preprocess_ratings_data(filepath="datasets/ratings.csv"):
    """
    Preprocess the ratings dataset.
    - Removes duplicate user-book pairs
    - Handles missing values
    - Normalizes ratings from 1-5 to 0-1 scale
    Returns: cleaned pandas DataFrame
    """
    df = pd.read_csv(filepath)
    print(f"Original shape: {df.shape}")

    df = df.drop_duplicates(subset=['user_id', 'book_id'])
    print(f"After removing duplicates: {df.shape}")

    df = df.dropna()

    scaler = MinMaxScaler()
    df['rating_normalized'] = scaler.fit_transform(df[['rating']])

    print(f"Final shape: {df.shape}")
    return df

def preprocess_sentiment_data(filepath="datasets/Customer_Sentiment.csv"):
    """
    Preprocess the customer sentiment dataset.
    - Removes duplicates
    - Handles missing values
    - Encodes categorical columns (gender, region, sentiment etc)
    - Normalizes customer_rating to 0-1 scale
    Returns: cleaned pandas DataFrame
    """
    df = pd.read_csv(filepath)
    print(f"Original shape: {df.shape}")

    df = df.drop_duplicates()
    print(f"After removing duplicates: {df.shape}")

    df = df.dropna()

    categorical_cols = ['gender', 'age_group', 'region', 
                       'product_category', 'purchase_channel', 
                       'platform', 'sentiment']
    le = LabelEncoder()
    for col in categorical_cols:
        if col in df.columns:
            df[col] = le.fit_transform(df[col].astype(str))

    scaler = MinMaxScaler()
    df['rating_normalized'] = scaler.fit_transform(
        df[['customer_rating']])

    print(f"Final shape: {df.shape}")
    return df

if __name__ == "__main__":
    print("=== Preprocessing Books Data ===")
    books_df = preprocess_books_data()
    
    print("\n=== Preprocessing Ratings Data ===")
    ratings_df = preprocess_ratings_data()
    
    print("\n=== Preprocessing Sentiment Data ===")
    sentiment_df = preprocess_sentiment_data()
    
    print("\n✅ All datasets preprocessed successfully!")
    print(f"Books: {books_df.shape}")
    print(f"Ratings: {ratings_df.shape}")
    print(f"Sentiment: {sentiment_df.shape}")