diff --git a/README.md b/README.md index 52f5458..eaf50a7 100644 --- a/README.md +++ b/README.md @@ -405,6 +405,20 @@ plot_features_interaction(data, "feature_1", "feature_2") | **Boolean** | ![](https://raw.githubusercontent.com/idanmoradarthas/DataScienceUtils/master/tests/baseline_images/test_preprocess/test_plot_relationship_between_features/test_plot_relationship_between_features_numeric_boolean.png) | ![](https://raw.githubusercontent.com/idanmoradarthas/DataScienceUtils/master/tests/baseline_images/test_preprocess/test_plot_relationship_between_features/test_plot_relationship_between_features_categorical_bool.png) | ![](https://raw.githubusercontent.com/idanmoradarthas/DataScienceUtils/master/tests/baseline_images/test_preprocess/test_plot_relationship_between_features/test_plot_relationship_between_features_both_bool.png) | | | **Datetime** | ![](https://raw.githubusercontent.com/idanmoradarthas/DataScienceUtils/master/tests/baseline_images/test_preprocess/test_plot_relationship_between_features/test_plot_relationship_between_features_datetime_numeric.png) | ![](https://raw.githubusercontent.com/idanmoradarthas/DataScienceUtils/master/tests/baseline_images/test_preprocess/test_plot_relationship_between_features/test_plot_relationship_between_features_datetime_categorical.png) | ![](https://raw.githubusercontent.com/idanmoradarthas/DataScienceUtils/master/tests/baseline_images/test_preprocess/test_plot_relationship_between_features/test_plot_relationship_between_features_datetime_bool_default.png) | ![](https://raw.githubusercontent.com/idanmoradarthas/DataScienceUtils/master/tests/baseline_images/test_preprocess/test_plot_relationship_between_features/test_plot_relationship_between_features_datetime_datetime.png) | +### Plot PCA Explained Variance + +This method plots the cumulative explained variance ratio of PCA components. It helps users quickly determine how many principal components are required to capture a desired proportion of variance in the data. +Horizontal reference lines are drawn at 70% and 80% variance. + +```python +from ds_utils.preprocess.visualization import plot_pca_explained_variance + +# Pass a dataframe containing only numerical features +plot_pca_explained_variance(data_numerical, use_scaling=True) +``` + +![Plot PCA Explained Variance](https://raw.githubusercontent.com/idanmoradarthas/DataScienceUtils/master/tests/baseline_images/test_preprocess/test_plot_pca_explained_variance/test_plot_pca_explained_variance_default.png) + ### Extract Statistics DataFrame per Label This method calculates comprehensive statistical metrics for numerical features grouped by label values. Use this when diff --git a/docs/source/preprocess/visualization.rst b/docs/source/preprocess/visualization.rst index affb5a0..5cefd0d 100644 --- a/docs/source/preprocess/visualization.rst +++ b/docs/source/preprocess/visualization.rst @@ -296,5 +296,41 @@ Choosing the Right Visualization - Use `get_correlated_features` and `visualize_correlations` to understand relationships between multiple features. - Use `plot_correlation_dendrogram` for a hierarchical view of feature relationships, especially useful for high-dimensional data. - Use `plot_features_interaction` to deep dive into the relationship between specific feature pairs. +- Use `plot_pca_explained_variance` to determine how many principal components are required to capture a desired proportion of variance. By combining these visualizations, you can gain a comprehensive understanding of your dataset's structure, which is crucial for effective data preprocessing, feature engineering, and model selection. + +*************************** +Plot PCA Explained Variance +*************************** + +This method visualizes the cumulative explained variance ratio of PCA components. Use this when you want to: + +- Determine how many principal components are required to capture a desired proportion of the total variance in the data. +- Perform dimensionality reduction using PCA. +- Understand how variance is distributed across the components. + +.. autofunction:: ds_utils.preprocess.visualization.plot_pca_explained_variance + +Code Example +============ + +Here's how to use the code:: + + import pandas as pd + from matplotlib import pyplot as plt + from ds_utils.preprocess.visualization import plot_pca_explained_variance + + data = pd.read_csv('path/to/dataset') + # Use only numeric features + numeric_data = data.select_dtypes(include="number") + + plot_pca_explained_variance(numeric_data, use_scaling=True) + plt.show() + +The plot displays the cumulative variance ratio as a line graph, with horizontal reference lines at 70% and 80% variance. + +.. image:: ../../../tests/baseline_images/test_preprocess/test_plot_pca_explained_variance/test_plot_pca_explained_variance_default.png + :align: center + :alt: Plot PCA Explained Variance + diff --git a/ds_utils/__init__.py b/ds_utils/__init__.py index f8aed37..e935881 100644 --- a/ds_utils/__init__.py +++ b/ds_utils/__init__.py @@ -1,3 +1,3 @@ """Data Science Utilities package.""" -__version__ = "1.10.0rc12" +__version__ = "1.10.0rc13" diff --git a/ds_utils/preprocess/visualization.py b/ds_utils/preprocess/visualization.py index c318ae9..c962ba6 100644 --- a/ds_utils/preprocess/visualization.py +++ b/ds_utils/preprocess/visualization.py @@ -15,7 +15,9 @@ from scipy.cluster.hierarchy import dendrogram, linkage from scipy.spatial.distance import squareform import seaborn as sns - +from sklearn.base import TransformerMixin +from sklearn.decomposition import PCA +from sklearn.preprocessing import StandardScaler from ds_utils.preprocess._plot_categorical import ( _plot_categorical_feature1, _plot_categorical_vs_numeric, @@ -246,3 +248,74 @@ def plot_features_interaction( ax = _plot_numeric_features(feature_1, feature_2, plot_data, remove_na, ax, **kwargs) return ax + + +def plot_pca_explained_variance( + X: pd.DataFrame, + use_scaling: bool = True, + scaler: Optional[TransformerMixin] = None, + legend_loc: str = "lower right", + ax: Optional[axes.Axes] = None, + pca_kwargs: Optional[dict] = None, + **kwargs, +) -> axes.Axes: + """Plot the cumulative explained variance ratio of PCA components. + + This visualization helps determine how many principal components are needed + to capture a desired proportion of the total variance in the data. + Horizontal reference lines are drawn at 70% and 80% variance. + + :param X: Input data with numerical features (rows = samples, columns = features). + :param use_scaling: If True, scale the data using the provided scaler before fitting PCA. + :param scaler: Scaler instance to use when use_scaling is True. If None, StandardScaler is used. + :param legend_loc: Location of the legend. Default is "lower right". + :param ax: Matplotlib Axes to draw the plot on. If None, a new figure and Axes are created. + :param pca_kwargs: Additional keyword arguments passed directly to sklearn.decomposition.PCA + (e.g., ``pca_kwargs={"n_components": 5}``). If None, PCA is initialized + with its defaults. + :param kwargs: Additional keyword arguments passed to axes.plot. + :return: The Axes object containing the plot. + :raises ValueError: If any column in X is non-numeric. + """ + if ax is None: + _, ax = plt.subplots(figsize=(8, 5)) + + if not np.all(X.dtypes.apply(pd.api.types.is_numeric_dtype)): + raise ValueError("All columns in X must be numeric.") + + X_array = X.to_numpy() + + if use_scaling: + _scaler = scaler if scaler is not None else StandardScaler() + X_array = _scaler.fit_transform(X_array) + + if pca_kwargs is None: + pca_kwargs = {} + + pca = PCA(**pca_kwargs) + pca.fit(X_array) + + explained_variance_ratio = pca.explained_variance_ratio_ + cumulative_variance = np.cumsum(explained_variance_ratio) + + ax.plot( + range(1, len(cumulative_variance) + 1), + cumulative_variance, + marker="o", + linestyle="-", + color="b", + label="Cumulative explained variance", + **kwargs, + ) + + # Reference lines for common variance thresholds + ax.axhline(0.70, color="gray", linestyle="--", linewidth=1, label="70% variance") + ax.axhline(0.80, color="gray", linestyle="--", linewidth=1, label="80% variance") + + ax.set_xlabel("Number of Principal Components") + ax.set_ylabel("Cumulative Explained Variance Ratio") + ax.set_title("PCA - Cumulative Explained Variance") + ax.grid(True) + ax.legend(loc=legend_loc) + + return ax diff --git a/pyproject.toml b/pyproject.toml index e4cd7a0..ba01857 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -56,17 +56,17 @@ path = "ds_utils/__init__.py" [project.optional-dependencies] test = [ - "pyarrow==23.0.1", + "pyarrow==24.0.0", "pytest==9.0.3", "pytest-mock==3.15.1", "pytest-cov==7.1.0", "pytest-xdist==3.8.0", "pytest-mpl==0.19.0", - "kaleido==1.2.0", - "ruff==0.15.11" + "kaleido==1.3.0", + "ruff==0.15.13" ] dev = [ - "build==1.4.3", + "build==1.5.0", "twine==6.2.0", ] nlp = [ @@ -76,15 +76,15 @@ nlp = [ [dependency-groups] dev = [ - "pyarrow==23.0.1", + "pyarrow==24.0.0", "pytest==9.0.3", "pytest-mock==3.15.1", "pytest-cov==7.1.0", "pytest-xdist==3.8.0", "pytest-mpl==0.19.0", - "kaleido==1.2.0", - "ruff==0.15.11", - "build==1.4.3", + "kaleido==1.3.0", + "ruff==0.15.13", + "build==1.5.0", "twine==6.2.0", ] diff --git a/skills/preprocess/SKILL.md b/skills/preprocess/SKILL.md index b267434..834495e 100644 --- a/skills/preprocess/SKILL.md +++ b/skills/preprocess/SKILL.md @@ -28,6 +28,7 @@ from ds_utils.preprocess.visualization import visualize_feature from ds_utils.preprocess.visualization import visualize_correlations from ds_utils.preprocess.visualization import plot_correlation_dendrogram from ds_utils.preprocess.visualization import plot_features_interaction +from ds_utils.preprocess.visualization import plot_pca_explained_variance from ds_utils.preprocess.statistics import get_correlated_features from ds_utils.preprocess.statistics import extract_statistics_dataframe_per_label from ds_utils.preprocess.statistics import compute_mutual_information @@ -150,6 +151,35 @@ plot_features_interaction(df, "feature1", "feature2", remove_na=False) --- +## plot_pca_explained_variance + +Plots the cumulative explained variance ratio of PCA components. + +```python +from ds_utils.preprocess.visualization import plot_pca_explained_variance + +# complete usage example +plot_pca_explained_variance(df, use_scaling=True) + +# limit the number of components +plot_pca_explained_variance(df, pca_kwargs={"n_components": 5}) +``` + +**Parameters:** +- `X` — pd.DataFrame, The dataset with numerical features. +- `use_scaling` — bool, Whether to scale the data before fitting PCA. +- `scaler` — TransformerMixin, Scaler instance to use when `use_scaling` is True (default: StandardScaler). +- `legend_loc` — str, Location of the legend (default: "lower right"). +- `ax` — axes.Axes, Matplotlib Axes to draw the plot on. +- `pca_kwargs` — dict, Additional keyword arguments passed directly to sklearn.decomposition.PCA. + +**Returns:** matplotlib Axes. + +**Common mistakes:** +- Passing a DataFrame with non-numeric columns. + +--- + ## get_correlated_features Extracts pairs of highly correlated features and their correlation to the target. @@ -243,4 +273,5 @@ visualize_feature(df["age"]) corr = df.corr() visualize_correlations(corr) results = get_correlated_features(corr, ["age", "income"], "is_churn") +plot_pca_explained_variance(df[["age", "income", "balance"]]) ``` diff --git a/tests/baseline_images/test_preprocess/test_plot_pca_explained_variance/test_plot_pca_explained_variance_ax_pass_through.png b/tests/baseline_images/test_preprocess/test_plot_pca_explained_variance/test_plot_pca_explained_variance_ax_pass_through.png new file mode 100644 index 0000000..4fbb37b Binary files /dev/null and b/tests/baseline_images/test_preprocess/test_plot_pca_explained_variance/test_plot_pca_explained_variance_ax_pass_through.png differ diff --git a/tests/baseline_images/test_preprocess/test_plot_pca_explained_variance/test_plot_pca_explained_variance_custom_scaler.png b/tests/baseline_images/test_preprocess/test_plot_pca_explained_variance/test_plot_pca_explained_variance_custom_scaler.png new file mode 100644 index 0000000..8005a48 Binary files /dev/null and b/tests/baseline_images/test_preprocess/test_plot_pca_explained_variance/test_plot_pca_explained_variance_custom_scaler.png differ diff --git a/tests/baseline_images/test_preprocess/test_plot_pca_explained_variance/test_plot_pca_explained_variance_default.png b/tests/baseline_images/test_preprocess/test_plot_pca_explained_variance/test_plot_pca_explained_variance_default.png new file mode 100644 index 0000000..dbc2920 Binary files /dev/null and b/tests/baseline_images/test_preprocess/test_plot_pca_explained_variance/test_plot_pca_explained_variance_default.png differ diff --git a/tests/baseline_images/test_preprocess/test_plot_pca_explained_variance/test_plot_pca_explained_variance_kwargs.png b/tests/baseline_images/test_preprocess/test_plot_pca_explained_variance/test_plot_pca_explained_variance_kwargs.png new file mode 100644 index 0000000..5c36497 Binary files /dev/null and b/tests/baseline_images/test_preprocess/test_plot_pca_explained_variance/test_plot_pca_explained_variance_kwargs.png differ diff --git a/tests/baseline_images/test_preprocess/test_plot_pca_explained_variance/test_plot_pca_explained_variance_no_scaling.png b/tests/baseline_images/test_preprocess/test_plot_pca_explained_variance/test_plot_pca_explained_variance_no_scaling.png new file mode 100644 index 0000000..d91b4b5 Binary files /dev/null and b/tests/baseline_images/test_preprocess/test_plot_pca_explained_variance/test_plot_pca_explained_variance_no_scaling.png differ diff --git a/tests/test_preprocess/test_plot_pca_explained_variance.py b/tests/test_preprocess/test_plot_pca_explained_variance.py new file mode 100644 index 0000000..6916a6c --- /dev/null +++ b/tests/test_preprocess/test_plot_pca_explained_variance.py @@ -0,0 +1,92 @@ +"""Tests for the plot_pca_explained_variance function.""" + +from pathlib import Path + +import pandas as pd +import pytest +from matplotlib import pyplot as plt +from sklearn.datasets import make_classification +from sklearn.preprocessing import MinMaxScaler + +from ds_utils.preprocess.visualization import plot_pca_explained_variance + +# Directory to store and load pytest-mpl baseline images for this test file +# The path is constructed to mirror the package structure inside the tests folder. +BASELINE_DIR = Path(__file__).parents[1] / "baseline_images" / Path(__file__).parent.name / Path(__file__).stem + + +@pytest.fixture +def sample_numeric_data(): + """Return a simple numeric DataFrame for PCA.""" + # We use n_redundant=5 to ensure the PCA results in an "elbow" curve, + # making for a more realistic and visually meaningful test. + X, _ = make_classification(n_samples=200, n_features=15, n_informative=5, n_redundant=5, random_state=42) + return pd.DataFrame(X, columns=[f"feature_{i}" for i in range(15)]) + + +@pytest.mark.mpl_image_compare(baseline_dir=BASELINE_DIR) +def test_plot_pca_explained_variance_default(sample_numeric_data): + """Test plot_pca_explained_variance with default parameters.""" + plot_pca_explained_variance(sample_numeric_data) + return plt.gcf() + + +@pytest.mark.mpl_image_compare(baseline_dir=BASELINE_DIR) +def test_plot_pca_explained_variance_no_scaling(sample_numeric_data): + """Test plot_pca_explained_variance without scaling.""" + plot_pca_explained_variance(sample_numeric_data, use_scaling=False) + return plt.gcf() + + +@pytest.mark.mpl_image_compare(baseline_dir=BASELINE_DIR) +def test_plot_pca_explained_variance_custom_scaler(sample_numeric_data): + """Test plot_pca_explained_variance with a custom scaler.""" + plot_pca_explained_variance(sample_numeric_data, use_scaling=True, scaler=MinMaxScaler()) + return plt.gcf() + + +@pytest.mark.mpl_image_compare(baseline_dir=BASELINE_DIR) +def test_plot_pca_explained_variance_kwargs(sample_numeric_data): + """Test plot_pca_explained_variance with extra kwargs and specific legend location.""" + ax = plot_pca_explained_variance(sample_numeric_data, legend_loc="upper left", linewidth=2, alpha=0.8) + + # "upper left" is mapped to location code 2 in matplotlib + assert ax.get_legend() is not None + assert ax.get_legend()._loc == 2 + return plt.gcf() + + +@pytest.mark.mpl_image_compare(baseline_dir=BASELINE_DIR) +def test_plot_pca_explained_variance_ax_pass_through(sample_numeric_data): + """Test plot_pca_explained_variance with a passed ax parameter. + + Note: The figsize=(10, 6) here is intentional for testing ax pass-through, + meaning the baseline image will have different dimensions than the defaults. + """ + fig, ax = plt.subplots(figsize=(10, 6)) + ax.set_title("Custom Ax Title") + + result_ax = plot_pca_explained_variance(sample_numeric_data, ax=ax) + + # Assert that the returned ax is exactly the one we passed in + assert result_ax is ax + # The function intentionally overwrites the title, so we assert it changed + assert result_ax.get_title() == "PCA - Cumulative Explained Variance" + + return fig + + +def test_plot_pca_explained_variance_pca_kwargs(sample_numeric_data): + """Test that pca_kwargs are forwarded to PCA correctly.""" + ax = plot_pca_explained_variance(sample_numeric_data, pca_kwargs={"n_components": 5}) + # Index 0 is the cumulative variance line; axhlines are added after in the implementation + line = ax.get_lines()[0] + assert len(line.get_xdata()) == 5 + plt.close() + + +def test_plot_pca_explained_variance_invalid_data(): + """Test plot_pca_explained_variance raises ValueError with non-numeric data.""" + df = pd.DataFrame({"A": [1, 2, 3], "B": ["x", "y", "z"]}) + with pytest.raises(ValueError, match="All columns in X must be numeric."): + plot_pca_explained_variance(df) diff --git a/tests/test_version.py b/tests/test_version.py index f8879c4..5724051 100644 --- a/tests/test_version.py +++ b/tests/test_version.py @@ -5,4 +5,4 @@ def test_version(): """Test that the package version is correct.""" - assert ds_utils.__version__ == "1.10.0rc12" + assert ds_utils.__version__ == "1.10.0rc13"