Update dependencies, add PCA explained variance plotting functionalit…

README.md

@@ -405,6 +405,20 @@ plot_features_interaction(data, "feature_1", "feature_2")
 | **Boolean**     | ![](https://raw.githubusercontent.com/idanmoradarthas/DataScienceUtils/master/tests/baseline_images/test_preprocess/test_plot_relationship_between_features/test_plot_relationship_between_features_numeric_boolean.png)     | ![](https://raw.githubusercontent.com/idanmoradarthas/DataScienceUtils/master/tests/baseline_images/test_preprocess/test_plot_relationship_between_features/test_plot_relationship_between_features_categorical_bool.png)     | ![](https://raw.githubusercontent.com/idanmoradarthas/DataScienceUtils/master/tests/baseline_images/test_preprocess/test_plot_relationship_between_features/test_plot_relationship_between_features_both_bool.png)             |                                                                                                                                                                                    |
 | **Datetime**    | ![](https://raw.githubusercontent.com/idanmoradarthas/DataScienceUtils/master/tests/baseline_images/test_preprocess/test_plot_relationship_between_features/test_plot_relationship_between_features_datetime_numeric.png)    | ![](https://raw.githubusercontent.com/idanmoradarthas/DataScienceUtils/master/tests/baseline_images/test_preprocess/test_plot_relationship_between_features/test_plot_relationship_between_features_datetime_categorical.png) | ![](https://raw.githubusercontent.com/idanmoradarthas/DataScienceUtils/master/tests/baseline_images/test_preprocess/test_plot_relationship_between_features/test_plot_relationship_between_features_datetime_bool_default.png) | ![](https://raw.githubusercontent.com/idanmoradarthas/DataScienceUtils/master/tests/baseline_images/test_preprocess/test_plot_relationship_between_features/test_plot_relationship_between_features_datetime_datetime.png) |
 
+### Plot PCA Explained Variance
+
+This method plots the cumulative explained variance ratio of PCA components. It helps users quickly determine how many principal components are required to capture a desired proportion of variance in the data.
+Horizontal reference lines are drawn at 70% and 80% variance.
+
+```python
+from ds_utils.preprocess.visualization import plot_pca_explained_variance
+
+# Pass a dataframe containing only numerical features
+plot_pca_explained_variance(data_numerical, use_scaling=True)
+```
+
+![Plot PCA Explained Variance](https://raw.githubusercontent.com/idanmoradarthas/DataScienceUtils/master/tests/baseline_images/test_preprocess/test_plot_pca_explained_variance/test_plot_pca_explained_variance_default.png)
+
 ### Extract Statistics DataFrame per Label
 
 This method calculates comprehensive statistical metrics for numerical features grouped by label values. Use this when

docs/source/preprocess/visualization.rst

@@ -296,5 +296,41 @@ Choosing the Right Visualization
 - Use `get_correlated_features` and `visualize_correlations` to understand relationships between multiple features.
 - Use `plot_correlation_dendrogram` for a hierarchical view of feature relationships, especially useful for high-dimensional data.
 - Use `plot_features_interaction` to deep dive into the relationship between specific feature pairs.
+- Use `plot_pca_explained_variance` to determine how many principal components are required to capture a desired proportion of variance.
 
 By combining these visualizations, you can gain a comprehensive understanding of your dataset's structure, which is crucial for effective data preprocessing, feature engineering, and model selection.
+
+***************************
+Plot PCA Explained Variance
+***************************
+
+This method visualizes the cumulative explained variance ratio of PCA components. Use this when you want to:
+
+- Determine how many principal components are required to capture a desired proportion of the total variance in the data.
+- Perform dimensionality reduction using PCA.
+- Understand how variance is distributed across the components.
+
+.. autofunction:: ds_utils.preprocess.visualization.plot_pca_explained_variance
+
+Code Example
+============
+
+Here's how to use the code::
+
+    import pandas as pd
+    from matplotlib import pyplot as plt
+    from ds_utils.preprocess.visualization import plot_pca_explained_variance
+
+    data = pd.read_csv('path/to/dataset')
+    # Use only numeric features
+    numeric_data = data.select_dtypes(include="number")
+
+    plot_pca_explained_variance(numeric_data, use_scaling=True)
+    plt.show()
+
+The plot displays the cumulative variance ratio as a line graph, with horizontal reference lines at 70% and 80% variance.
+
+.. image:: ../../../tests/baseline_images/test_preprocess/test_plot_pca_explained_variance/test_plot_pca_explained_variance_default.png
+    :align: center
+    :alt: Plot PCA Explained Variance
+

ds_utils/__init__.py

@@ -1,3 +1,3 @@
 """Data Science Utilities package."""
 
-__version__ = "1.10.0rc12"
+__version__ = "1.10.0rc13"

ds_utils/preprocess/visualization.py

@@ -15,7 +15,9 @@
 from scipy.cluster.hierarchy import dendrogram, linkage
 from scipy.spatial.distance import squareform
 import seaborn as sns
-
+from sklearn.base import TransformerMixin
+from sklearn.decomposition import PCA
+from sklearn.preprocessing import StandardScaler
 from ds_utils.preprocess._plot_categorical import (
     _plot_categorical_feature1,
     _plot_categorical_vs_numeric,
@@ -246,3 +248,74 @@ def plot_features_interaction(
         ax = _plot_numeric_features(feature_1, feature_2, plot_data, remove_na, ax, **kwargs)
 
     return ax
+
+
+def plot_pca_explained_variance(
+    X: pd.DataFrame,
+    use_scaling: bool = True,
+    scaler: Optional[TransformerMixin] = None,
+    legend_loc: str = "lower right",
+    ax: Optional[axes.Axes] = None,
+    pca_kwargs: Optional[dict] = None,
+    **kwargs,
+) -> axes.Axes:
+    """Plot the cumulative explained variance ratio of PCA components.
+
+    This visualization helps determine how many principal components are needed
+    to capture a desired proportion of the total variance in the data.
+    Horizontal reference lines are drawn at 70% and 80% variance.
+
+    :param X: Input data with numerical features (rows = samples, columns = features).
+    :param use_scaling: If True, scale the data using the provided scaler before fitting PCA.
+    :param scaler: Scaler instance to use when use_scaling is True. If None, StandardScaler is used.
+    :param legend_loc: Location of the legend. Default is "lower right".
+    :param ax: Matplotlib Axes to draw the plot on. If None, a new figure and Axes are created.
+    :param pca_kwargs: Additional keyword arguments passed directly to sklearn.decomposition.PCA
+                       (e.g., ``pca_kwargs={"n_components": 5}``). If None, PCA is initialized
+                       with its defaults.
+    :param kwargs: Additional keyword arguments passed to axes.plot.
+    :return: The Axes object containing the plot.
+    :raises ValueError: If any column in X is non-numeric.
+    """
+    if ax is None:
+        _, ax = plt.subplots(figsize=(8, 5))
+
+    if not np.all(X.dtypes.apply(pd.api.types.is_numeric_dtype)):
+        raise ValueError("All columns in X must be numeric.")
+
+    X_array = X.to_numpy()
+
+    if use_scaling:
+        _scaler = scaler if scaler is not None else StandardScaler()
+        X_array = _scaler.fit_transform(X_array)
+
+    if pca_kwargs is None:
+        pca_kwargs = {}
+
+    pca = PCA(**pca_kwargs)
+    pca.fit(X_array)
+
+    explained_variance_ratio = pca.explained_variance_ratio_
+    cumulative_variance = np.cumsum(explained_variance_ratio)
+
+    ax.plot(
+        range(1, len(cumulative_variance) + 1),
+        cumulative_variance,
+        marker="o",
+        linestyle="-",
+        color="b",
+        label="Cumulative explained variance",
+        **kwargs,
+    )
+
+    # Reference lines for common variance thresholds
+    ax.axhline(0.70, color="gray", linestyle="--", linewidth=1, label="70% variance")
+    ax.axhline(0.80, color="gray", linestyle="--", linewidth=1, label="80% variance")
+
+    ax.set_xlabel("Number of Principal Components")
+    ax.set_ylabel("Cumulative Explained Variance Ratio")
+    ax.set_title("PCA - Cumulative Explained Variance")
+    ax.grid(True)
+    ax.legend(loc=legend_loc)
+
+    return ax

pyproject.toml

@@ -56,17 +56,17 @@ path = "ds_utils/__init__.py"
 
 [project.optional-dependencies]
 test = [
-    "pyarrow==23.0.1",
+    "pyarrow==24.0.0",
     "pytest==9.0.3",
     "pytest-mock==3.15.1",
     "pytest-cov==7.1.0",
     "pytest-xdist==3.8.0",
     "pytest-mpl==0.19.0",
-    "kaleido==1.2.0",
-    "ruff==0.15.11"
+    "kaleido==1.3.0",
+    "ruff==0.15.13"
 ]
 dev = [
-    "build==1.4.3",
+    "build==1.5.0",
     "twine==6.2.0",
 ]
 nlp = [
@@ -76,15 +76,15 @@ nlp = [
 
 [dependency-groups]
 dev = [
-    "pyarrow==23.0.1",
+    "pyarrow==24.0.0",
     "pytest==9.0.3",
     "pytest-mock==3.15.1",
     "pytest-cov==7.1.0",
     "pytest-xdist==3.8.0",
     "pytest-mpl==0.19.0",
-    "kaleido==1.2.0",
-    "ruff==0.15.11",
-    "build==1.4.3",
+    "kaleido==1.3.0",
+    "ruff==0.15.13",
+    "build==1.5.0",
     "twine==6.2.0",
 ]
 

skills/preprocess/SKILL.md

@@ -28,6 +28,7 @@ from ds_utils.preprocess.visualization import visualize_feature
 from ds_utils.preprocess.visualization import visualize_correlations
 from ds_utils.preprocess.visualization import plot_correlation_dendrogram
 from ds_utils.preprocess.visualization import plot_features_interaction
+from ds_utils.preprocess.visualization import plot_pca_explained_variance
 from ds_utils.preprocess.statistics import get_correlated_features
 from ds_utils.preprocess.statistics import extract_statistics_dataframe_per_label
 from ds_utils.preprocess.statistics import compute_mutual_information
@@ -150,6 +151,35 @@ plot_features_interaction(df, "feature1", "feature2", remove_na=False)
 
 ---
 
+## plot_pca_explained_variance
+
+Plots the cumulative explained variance ratio of PCA components.
+
+```python
+from ds_utils.preprocess.visualization import plot_pca_explained_variance
+
+# complete usage example
+plot_pca_explained_variance(df, use_scaling=True)
+
+# limit the number of components
+plot_pca_explained_variance(df, pca_kwargs={"n_components": 5})
+```
+
+**Parameters:**
+- `X` — pd.DataFrame, The dataset with numerical features.
+- `use_scaling` — bool, Whether to scale the data before fitting PCA.
+- `scaler` — TransformerMixin, Scaler instance to use when `use_scaling` is True (default: StandardScaler).
+- `legend_loc` — str, Location of the legend (default: "lower right").
+- `ax` — axes.Axes, Matplotlib Axes to draw the plot on.
+- `pca_kwargs` — dict, Additional keyword arguments passed directly to sklearn.decomposition.PCA.
+
+**Returns:** matplotlib Axes.
+
+**Common mistakes:**
+- Passing a DataFrame with non-numeric columns.
+
+---
+
 ## get_correlated_features
 
 Extracts pairs of highly correlated features and their correlation to the target.
@@ -243,4 +273,5 @@ visualize_feature(df["age"])
 corr = df.corr()
 visualize_correlations(corr)
 results = get_correlated_features(corr, ["age", "income"], "is_churn")
+plot_pca_explained_variance(df[["age", "income", "balance"]])
 ```

tests/test_preprocess/test_plot_pca_explained_variance.py

@@ -0,0 +1,92 @@
+"""Tests for the plot_pca_explained_variance function."""
+
+from pathlib import Path
+
+import pandas as pd
+import pytest
+from matplotlib import pyplot as plt
+from sklearn.datasets import make_classification
+from sklearn.preprocessing import MinMaxScaler
+
+from ds_utils.preprocess.visualization import plot_pca_explained_variance
+
+# Directory to store and load pytest-mpl baseline images for this test file
+# The path is constructed to mirror the package structure inside the tests folder.
+BASELINE_DIR = Path(__file__).parents[1] / "baseline_images" / Path(__file__).parent.name / Path(__file__).stem
+
+
+@pytest.fixture
+def sample_numeric_data():
+    """Return a simple numeric DataFrame for PCA."""
+    # We use n_redundant=5 to ensure the PCA results in an "elbow" curve,
+    # making for a more realistic and visually meaningful test.
+    X, _ = make_classification(n_samples=200, n_features=15, n_informative=5, n_redundant=5, random_state=42)
+    return pd.DataFrame(X, columns=[f"feature_{i}" for i in range(15)])
+
+
+@pytest.mark.mpl_image_compare(baseline_dir=BASELINE_DIR)
+def test_plot_pca_explained_variance_default(sample_numeric_data):
+    """Test plot_pca_explained_variance with default parameters."""
+    plot_pca_explained_variance(sample_numeric_data)
+    return plt.gcf()
+
+
+@pytest.mark.mpl_image_compare(baseline_dir=BASELINE_DIR)
+def test_plot_pca_explained_variance_no_scaling(sample_numeric_data):
+    """Test plot_pca_explained_variance without scaling."""
+    plot_pca_explained_variance(sample_numeric_data, use_scaling=False)
+    return plt.gcf()
+
+
+@pytest.mark.mpl_image_compare(baseline_dir=BASELINE_DIR)
+def test_plot_pca_explained_variance_custom_scaler(sample_numeric_data):
+    """Test plot_pca_explained_variance with a custom scaler."""
+    plot_pca_explained_variance(sample_numeric_data, use_scaling=True, scaler=MinMaxScaler())
+    return plt.gcf()
+
+
+@pytest.mark.mpl_image_compare(baseline_dir=BASELINE_DIR)
+def test_plot_pca_explained_variance_kwargs(sample_numeric_data):
+    """Test plot_pca_explained_variance with extra kwargs and specific legend location."""
+    ax = plot_pca_explained_variance(sample_numeric_data, legend_loc="upper left", linewidth=2, alpha=0.8)
+
+    # "upper left" is mapped to location code 2 in matplotlib
+    assert ax.get_legend() is not None
+    assert ax.get_legend()._loc == 2
+    return plt.gcf()
+
+
+@pytest.mark.mpl_image_compare(baseline_dir=BASELINE_DIR)
+def test_plot_pca_explained_variance_ax_pass_through(sample_numeric_data):
+    """Test plot_pca_explained_variance with a passed ax parameter.
+
+    Note: The figsize=(10, 6) here is intentional for testing ax pass-through,
+    meaning the baseline image will have different dimensions than the defaults.
+    """
+    fig, ax = plt.subplots(figsize=(10, 6))
+    ax.set_title("Custom Ax Title")
+
+    result_ax = plot_pca_explained_variance(sample_numeric_data, ax=ax)
+
+    # Assert that the returned ax is exactly the one we passed in
+    assert result_ax is ax
+    # The function intentionally overwrites the title, so we assert it changed
+    assert result_ax.get_title() == "PCA - Cumulative Explained Variance"
+
+    return fig
+
+
+def test_plot_pca_explained_variance_pca_kwargs(sample_numeric_data):
+    """Test that pca_kwargs are forwarded to PCA correctly."""
+    ax = plot_pca_explained_variance(sample_numeric_data, pca_kwargs={"n_components": 5})
+    # Index 0 is the cumulative variance line; axhlines are added after in the implementation
+    line = ax.get_lines()[0]
+    assert len(line.get_xdata()) == 5
+    plt.close()
+
+
+def test_plot_pca_explained_variance_invalid_data():
+    """Test plot_pca_explained_variance raises ValueError with non-numeric data."""
+    df = pd.DataFrame({"A": [1, 2, 3], "B": ["x", "y", "z"]})
+    with pytest.raises(ValueError, match="All columns in X must be numeric."):
+        plot_pca_explained_variance(df)

tests/test_version.py

@@ -5,4 +5,4 @@
 
 def test_version():
     """Test that the package version is correct."""
-    assert ds_utils.__version__ == "1.10.0rc12"
+    assert ds_utils.__version__ == "1.10.0rc13"