From bf9707b36959950aa3d5c9646998bf17f6fbe4cb Mon Sep 17 00:00:00 2001 From: Eton Tackett Date: Fri, 20 Mar 2026 18:45:54 -0500 Subject: [PATCH 01/21] Changes --- test/runtests.jl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/runtests.jl b/test/runtests.jl index dbbe06f..209d515 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -2,5 +2,6 @@ using RidgeRegression using Test @testset "RidgeRegression.jl" begin - # Write your tests here. + include("dataset_tests.jl") + include("bidiagonalization_tests.jl") end From 214e74d0fcabf336b87f7528502ae39c3b615cf3 Mon Sep 17 00:00:00 2001 From: Eton Tackett Date: Fri, 20 Mar 2026 18:47:02 -0500 Subject: [PATCH 02/21] Ridge Regreesion jl changes --- src/RidgeRegression.jl | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/RidgeRegression.jl b/src/RidgeRegression.jl index 5ef574e..53f2cde 100644 --- a/src/RidgeRegression.jl +++ b/src/RidgeRegression.jl @@ -3,9 +3,8 @@ using CSV using DataFrames using Downloads -export Dataset, csv_dataset, one_hot_encode - include("dataset.jl") +export Dataset, csv_dataset, one_hot_encode end From a87c905508ea53799a632521b50802b5206a9769 Mon Sep 17 00:00:00 2001 From: Eton Tackett Date: Sat, 21 Mar 2026 15:15:16 -0500 Subject: [PATCH 03/21] Fix test environment and dataset tests --- Project.toml | 1 + src/RidgeRegression.jl | 1 + test/Project.toml | 7 +++++++ test/dataset_tests.jl | 1 - test/runtests.jl | 1 - 5 files changed, 9 insertions(+), 2 deletions(-) diff --git a/Project.toml b/Project.toml index 427e8aa..9c27dee 100644 --- a/Project.toml +++ b/Project.toml @@ -7,6 +7,7 @@ authors = ["Eton Tackett ", "Vivak Patel Date: Sat, 21 Mar 2026 15:34:04 -0500 Subject: [PATCH 04/21] Remove design.md from this PR (separate design branch) --- docs/src/design.md | 68 ---------------------------------------------- 1 file changed, 68 deletions(-) delete mode 100644 docs/src/design.md diff --git a/docs/src/design.md b/docs/src/design.md deleted file mode 100644 index 4795866..0000000 --- a/docs/src/design.md +++ /dev/null @@ -1,68 +0,0 @@ -# Motivation and Background -Many modern science problems involve regression problems with extremely large numbers of predictors. Genome-wide association studies (GWAS), for example, try to identify genetic variants associated with a disease phenotype using hundreds of thousands or millions of genomic features. In such settings, traditional least squares methods fail because noise and ill-conditioning. Penalized Least Squares (PLS) extends ordinary least squares (OLS) regression by adding a penalty term to shrink parameter estimates. Ridge regression, an approach within PLS, adds a regularization term, producing a regularized estimator that helps to stabalize the solution. - -There are many numerical algorithms available to compute ridge regression estimates including direct methods, Krylov subspace methods, gradient-based optimization, coordinate descent, and stochastic gradient descent. These algorithms differ in their computational cost, memory requirements, and numerical stability. - -The goal of this experiment is to investigate the performance of these algorithms when we vary the structure and scale of the regression problem. To do this, we consider the linear model $\mathbf{y} = X\boldsymbol{\beta} + \boldsymbol{\varepsilon}$ where the matrix ${X}$ will be constructed with varying dimensions, sparsity patterns, and conditioning properties. -# Questions -Key Questions: -Which ridge regression algorithm is provides the best balance between: - -- Numerical stability -- Computational aspects (GPU/CPU, runtime, etc) -# Experimental Units -The experimental units are the datasets under fixed penalty weights. Each dataset will contain a matrix ${X}$, a response vector $\mathbf{y}$, and a regularization parameter ${\lambda}$. Because the statistical behavior of ridge regression algorithms depends strongly on the dimensional structure of the problem, a blocking system will be used. Datasets will be grouped according to their dimensional regime, characterized as $p \ll n$, p ≈ n, and $p \gg n$. These regimes correspond to fundamentally different geometric properties of the design matrix, including rank behavior, conditioning, and the stability of the normal equations. - -In addition to dimensional block, the strength of the ridge penalty will be incorporated as a secondary blocking factor. The ridge estimator is $\hat{\beta_R} = (X^\top X + \lambda I)^{-1}X^\top y$. The matrix conditioning number is defined as $\kappa(A) = \frac{\sigma_{\max}(A)}{\sigma_{\min}(A)}$. In the context of ridge regression, the regularization parameter ${\lambda}$, can impact the conditioning number. Let $X = U\Sigma V^\top$ be the SVD of $X$, with singular values $\sigma_1,\dots,\sigma_p$. - -Then - -$$ -X^\top X = V \Sigma^\top \Sigma V^\top -= V \,\mathrm{diag}(\sigma_1^2,\dots,\sigma_p^2)\, V^\top . -$$ - -Adding the ridge term gives - -$$ -X^\top X + \lambda I -= -V \,\mathrm{diag}(\sigma_1^2+\lambda,\dots,\sigma_p^2+\lambda)\, V^\top . -$$ -$$ -\kappa_2(X^\top X+\lambda I) -= -\frac{\sigma_{\max}^2+\lambda}{\sigma_{\min}^2+\lambda}. -$$ - -When ${\lambda}$ is small and the singular values are large, we aren't changing the conditioning number much. As such, the ridge penalty won't really affect the condition number. Conversely, if $\sigma_{\min}$ is close to 0, we have an ill-posed problem (Existence, uniqueness, stability not satisfied). However if a large ${\lambda}$ is chosen, the condition number is reduced and the problem becomes more stable. This behavior motivates blocking experiments according to the effective conditioning induced by the ridge penalty, allowing algorithm performance to be compared across well-posed and ill-posed regression settings. - -Another blocking factor that will be considered is how sparse or dense the matrix X is. Many algorithms behave differently depending on whether the matrix is sparse or dense. In ridge regression, there are many operations involving X including matrix-matrix products and matrix-vector products. A dense matrix leads to high computational cost whereas a sparse matrix we can significantly reduce the cost. As such, different algorithms may perform better depending on the sparsity structure of X, making matrix sparsity a relevant blocking factor when comparing algorithm behavior and computational efficiency. - -| Blocking System | Factor | Blocks | -|:----------------|:-------|:-------| -| Dataset | Dimensional regime (\(p/n\)) | $(p \ll n)$, $(p \approx n)$, $(p \gg n)$| -| Ridge Penalty| Value of ${\lambda}$ relative to the singular values | Small, Large (Frobenius Norm range of values, Calculate Singular Values)| -| Matrix Sparsity| Density of non-zero values in X | Sparse, Moderate, Dense (Need to ask about how to quantify this)| -# Treatments -(How many treatments should we expect) -(What treatments correspond to which blocking system) - -The treatments are the ridge regression solution methods: - -- Gradient-based optimization -- Stochastic gradient descent -- Direct Methods - -For each experimental unit, all treatments will be applied to the dataset. This will be done so that differences in performance can be attributed to the algorithms themselves rather than the data. The order of treatment application will be completely randomized to avoid any systemic bias. If there are b possible combinations of the levels of each blocking factor - -The total number of block combinations is determined by the product of the number of levels in each blocking factor. For example, if the experiment includes three dimensional regimes, two sparsity levels, and two regularization strengths, then there are $3 * 2 * 2 = 12$ block combinations. -# Observational Units and Measurements -Explanation needed -(How many rows/columns to expect) -(Explain measurements: Floating point, real, etc. Meaning of values, what values it can take.) -The observational units are each algorithm-dataset pair. For each combination we will observe the following -| Measurement System | Factor | Measurements | -|:--------------------------|:--------------------------|:-------------| -| Computational Performance | Efficiency | Runtime (seconds), Iterations to convergence | -| Numerical Stability | Solution accuracy | Perturbation sensitivity | \ No newline at end of file From 13fc5bf3df477ff1a53f97533c575c53784eea91 Mon Sep 17 00:00:00 2001 From: Eton Tackett Date: Sun, 22 Mar 2026 21:27:59 -0500 Subject: [PATCH 05/21] 3/22 Updates --- src/RidgeRegression.jl | 2 +- src/dataset.jl | 85 ++++++++++++++++++++++------------ test/dataset_tests.jl | 68 +++++---------------------- test/encoding_tests.jl | 38 +++++++++++++++ test/load_csv_dataset_tests.jl | 38 +++++++++++++++ test/runtests.jl | 14 +++++- 6 files changed, 156 insertions(+), 89 deletions(-) create mode 100644 test/encoding_tests.jl create mode 100644 test/load_csv_dataset_tests.jl diff --git a/src/RidgeRegression.jl b/src/RidgeRegression.jl index 011adc5..f1bbc19 100644 --- a/src/RidgeRegression.jl +++ b/src/RidgeRegression.jl @@ -6,6 +6,6 @@ using Downloads include("dataset.jl") -export Dataset, csv_dataset, one_hot_encode +export Dataset, load_csv_dataset, one_hot_encode end diff --git a/src/dataset.jl b/src/dataset.jl index 4e20c89..16b3246 100644 --- a/src/dataset.jl +++ b/src/dataset.jl @@ -1,29 +1,49 @@ """ - Dataset(name, X, y) + Dataset <: ExperimentalUnit -Contains datasets for ridge regression experiments. +A dataset for Ridge Regression experiements. + +# Description + +A `Dataset` object stores the design matrix ``X`` and response vector ``y`` +for a regression problem. These datasets serve as the experimental units for ridge regression experiments, allowing us to evaluate the performance of ridge regression models on various datasets. # Fields - `name::String`: Name of dataset -- `X::Matrix{Float64}`: Matrix of variables/features -- `y::Vector{Float64}`: Target vector +- `X::TX`: Matrix of variables/features +- `y::TY`: Target vector + +# Constructor + + Dataset(name::String, X::AbstractMatrix, y::AbstractVector) + +## Arguments +- `name::String`: Name of dataset +- `X::TX`: Matrix of variables/features +- `y::TY`: Target vector -# Throws +## Returns +- A `Dataset` object containing the numeric design matrix and response vector. + +## Throws - `ArgumentError`: If rows in `X` does not equal length of `y`. !!! note - Used as the experimental unit for ridge regression experiments. + `Dataset` objects are used as experimental units when evaluating + ridge regression algorithms. The parametric design allows both dense + and sparse matrices to be stored without forcing conversion to a + dense `Matrix{Float64}`. """ -struct Dataset +struct Dataset{TX<:AbstractMatrix, TY<:AbstractVector} name::String - X::Matrix{Float64} - y::Vector{Float64} + X::TX + y::TY - function Dataset(name::String, X::AbstractMatrix, y::AbstractVector) + function Dataset(name::String, X::TX, y::TY) where {TX<:AbstractMatrix, TY<:AbstractVector} size(X, 1) == length(y) || throw(ArgumentError("X and y must have same number of rows")) - new(name, Matrix{Float64}(X), Vector{Float64}(y)) + new{TX, TY}(name, X, y) end end @@ -36,28 +56,28 @@ One-hot encode categorical (string-like) features in `Xdf`. - `Xdf::DataFrame`: Input DataFrame containing features and response vector `y`. # Keyword Arguments +- `cols_to_encode`: A collection of column names or indices to one-hot encode. - `drop_first::Bool=true`: If `true`, drop the first dummy column for each categorical feature to avoid multicollinearity. # Returns - `Matrix{Float64}`: A numeric matrix containing the encoded feature. """ -function one_hot_encode(Xdf::DataFrame; drop_first::Bool = true)::Matrix{Float64} +function one_hot_encode(Xdf::DataFrame; cols_to_encode, drop_first::Bool = true)::Matrix{Float64} n = nrow(Xdf) cols = Vector{Vector{Float64}}() + encode_names = Set(c isa Int ? Symbol(names(Xdf)[c]) : Symbol(c) for c in cols_to_encode) + for name in names(Xdf) #Selecting columns that aren't the target variable and pushing them to the columns. col = Xdf[!, name] - if eltype(col) <: Real - push!(cols, Float64.(col)) - continue - end - - scol = string.(col) # Convert to string for categorical processing. - lv = unique(scol) #Get unique category levels. - ind = scol .== permutedims(lv) #Create indicator matrix for each level of the categorical variable. - #Permutedims is used to align the dimensions for broadcasting. - #Broadcasting compares each element of `scol` with each level in `lv`, resulting in a matrix where each column corresponds to a level and contains `true` for rows that match that level and `false` otherwise. + name_sym = Symbol(name) + if name_sym in encode_names + scol = string.(col) # Convert to string for categorical processing. + lv = unique(scol) #Get unique category levels. + ind = scol .== permutedims(lv) #Create indicator matrix for each level of the categorical variable. + #Permutedims is used to align the dimensions for broadcasting. + #Broadcasting compares each element of `scol` with each level in `lv`, resulting in a matrix where each column corresponds to a level and contains `true` for rows that match that level and `false` otherwise. if drop_first && size(ind, 2) > 1 #Drop the first column of the indicator matrix to avoid multicollinearity if drop_first is true and there are multiple levels. ind = ind[:, 2:end] @@ -66,6 +86,12 @@ function one_hot_encode(Xdf::DataFrame; drop_first::Bool = true)::Matrix{Float64 for j in 1:size(ind, 2) push!(cols, Float64.(ind[:, j])) #Convert the boolean indicator columns to Float64 and add them to the list of columns. end + else + eltype(col) <: Real || + throw(ArgumentError("Column $name must be numeric unless it is listed in cols_to_encode")) + + push!(cols, Float64.(col)) + end end p = length(cols) @@ -78,7 +104,7 @@ function one_hot_encode(Xdf::DataFrame; drop_first::Bool = true)::Matrix{Float64 end """ - csv_dataset(path_or_url; target_col, name="csv_dataset") + load_csv_dataset(path_or_url; target_col, name="csv_dataset") Load a dataset from a CSV file or URL. @@ -86,16 +112,14 @@ Load a dataset from a CSV file or URL. - `path_or_url::String`: Local file path or web URL containing CSV data. # Keyword Arguments +- `cols_to_encode=Symbol[]`: Column names or indices in the feature data to one-hot encode. - `target_col`: Column index or column name containing the response variable. - `name::String="csv_dataset"`: Dataset name. # Returns - `Dataset`: A dataset containing the encoded feature matrix `X`, response vector `y`, and dataset name. """ -function csv_dataset(path_or_url::String; - target_col, - name::String = "csv_dataset" -) +function load_csv_dataset(path_or_url::String; cols_to_encode=Symbol[], target_col, name::String = "csv_dataset") filepath = startswith(path_or_url, "http") ? @@ -111,9 +135,10 @@ function csv_dataset(path_or_url::String; df[:, Symbol(target_col)] #Extract the target variable based on whether target_col is an index or a name. - X = one_hot_encode(Xdf; drop_first = true) - + feature_names = names(Xdf) + encode_cols = [c isa Int ? Symbol(names(Xdf)[c]) : Symbol(c) for c in cols_to_encode] + X = one_hot_encode(Xdf; cols_to_encode=encode_cols, drop_first = true) - return Dataset(name, Matrix{Float64}(X), Vector{Float64}(y)) + return Dataset(name, X, collect(Float64, y)) end diff --git a/test/dataset_tests.jl b/test/dataset_tests.jl index 1d8ee80..a0176a9 100644 --- a/test/dataset_tests.jl +++ b/test/dataset_tests.jl @@ -1,65 +1,19 @@ -using Test -using DataFrames -using CSV -using RidgeRegression -@testset "Dataset" begin +@testset "Testset 1" begin X = [1 2; 3 4] y = [10, 20] d = Dataset("toy", X, y) - @test d.name == "toy" - @test d.X isa Matrix{Float64} - @test d.y isa Vector{Float64} - @test size(d.X) == (2, 2) - @test length(d.y) == 2 - @test d.X[1, 1] == 1.0 - @test d.y[2] == 20.0 - - @test_throws ArgumentError Dataset("bad", X, [1, 2, 3]) -end - -@testset "one_hot_encode" begin - df = DataFrame( - A = ["red", "blue", "red", "green"], - B = [1, 2, 3, 4], - C = ["small", "large", "medium", "small"] - ) - - X = redirect_stdout(devnull) do - one_hot_encode(df; drop_first = true) - end - - @test size(X) == (4, 5) - @test X[:, 3] == [1.0, 2.0, 3.0, 4.0] - @test all(x -> x == 0.0 || x == 1.0, X[:, [1,2,4,5]]) - @test all(vec(sum(X[:, 1:2]; dims=2)) .<= 1) - @test all(vec(sum(X[:, 4:5]; dims=2)) .<= 1) + @test "toy" == d.name + @test X == d.X + @test y == d.y + @test (2, 2) == size(d.X) + @test 2 == length(d.y) + @test 1.0 == d.X[1, 1] + @test 20.0 == d.y[2] end -@testset "csv_dataset" begin - tmp = tempname() * ".csv" - df = DataFrame( - a = [1.0, 2.0, missing, 4.0], - b = ["x", "y", "y", "x"], - y = [10.0, 20.0, 30.0, 40.0] - ) - CSV.write(tmp, df) - - d = redirect_stdout(devnull) do - csv_dataset(tmp; target_col=:y, name="tmp") - end - - @test d.name == "tmp" - @test d.X isa Matrix{Float64} - @test d.y isa Vector{Float64} - - @test length(d.y) == 3 - @test size(d.X, 1) == 3 - @test d.y == [10.0, 20.0, 40.0] +@testset "Testset 2" begin + X = [1 2; 3 4] - d2 = redirect_stdout(devnull) do - csv_dataset(tmp; target_col=3, name="tmp2") - end - @test d2.y == [10.0, 20.0, 40.0] - @test size(d2.X, 1) == 3 + @test_throws ArgumentError Dataset("bad", X, [1, 2, 3]) end diff --git a/test/encoding_tests.jl b/test/encoding_tests.jl new file mode 100644 index 0000000..ca67e61 --- /dev/null +++ b/test/encoding_tests.jl @@ -0,0 +1,38 @@ +@testset "Testset 1" begin + df = DataFrame( + A = ["red", "blue", "red", "green"], + B = [1, 2, 3, 4], + C = ["small", "large", "medium", "small"] + ) + + X = one_hot_encode(df; cols_to_encode=[:A, :C], drop_first=true) + + @test (4, 5) == size(X) + @test [1.0, 2.0, 3.0, 4.0] == X[:, 3] + @test all(x -> x == 0.0 || x == 1.0, X[:, [1, 2, 4, 5]]) + @test all(vec(sum(X[:, 1:2]; dims=2)) .<= 1) + @test all(vec(sum(X[:, 4:5]; dims=2)) .<= 1) +end + +@testset "Testset 2" begin + df = DataFrame( + A = ["red", "blue", "red", "green"], + B = [1, 2, 3, 4], + C = ["small", "large", "medium", "small"] + ) + + @test_throws ArgumentError one_hot_encode(df; cols_to_encode=[:A], drop_first=true) +end + +@testset "Testset 3" begin + df = DataFrame( + group = [1, 2, 1, 3], + x = [10.0, 20.0, 30.0, 40.0] + ) + + X = one_hot_encode(df; cols_to_encode=[:group], drop_first=true) + + @test (4, 3) == size(X) + @test [10.0, 20.0, 30.0, 40.0] == X[:, 3] + @test all(x -> x == 0.0 || x == 1.0, X[:, 1:2]) +end diff --git a/test/load_csv_dataset_tests.jl b/test/load_csv_dataset_tests.jl new file mode 100644 index 0000000..67f41ed --- /dev/null +++ b/test/load_csv_dataset_tests.jl @@ -0,0 +1,38 @@ +@testset "Testset 1" begin + tmp = tempname() * ".csv" + + df = DataFrame( + a = [1.0, 2.0, missing, 4.0], + b = ["x", "y", "y", "x"], + y = [10.0, 20.0, 30.0, 40.0] + ) + + CSV.write(tmp, df) + + d = load_csv_dataset(tmp; target_col=:y, cols_to_encode=[:b], name="tmp") + + @test "tmp" == d.name + @test 3 == length(d.y) + @test 3 == size(d.X, 1) + @test [10.0, 20.0, 40.0] == d.y + @test (3, 2) == size(d.X) +end + +@testset "Testset 2" begin + tmp = tempname() * ".csv" + + df = DataFrame( + a = [1.0, 2.0, missing, 4.0], + b = ["x", "y", "y", "x"], + y = [10.0, 20.0, 30.0, 40.0] + ) + + CSV.write(tmp, df) + + d = load_csv_dataset(tmp; target_col=3, cols_to_encode=[:b], name="tmp2") + + @test "tmp2" == d.name + @test [10.0, 20.0, 40.0] == d.y + @test 3 == size(d.X, 1) + @test (3, 2) == size(d.X) +end diff --git a/test/runtests.jl b/test/runtests.jl index 3154bcd..c8b78c5 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,6 +1,18 @@ using RidgeRegression using Test +using DataFrames +using CSV @testset "RidgeRegression.jl" begin - include("dataset_tests.jl") + @testset "Dataset tests" begin + include("dataset_tests.jl") + end + + @testset "Encoding tests" begin + include("encoding_tests.jl") + end + + @testset "Load CSV dataset tests" begin + include("load_csv_dataset_tests.jl") + end end From 5cbeb0b0ac5001a35841f49f02b6c56b1b227f09 Mon Sep 17 00:00:00 2001 From: Eton Tackett Date: Tue, 24 Mar 2026 10:08:16 -0500 Subject: [PATCH 06/21] New Edits --- test/runtests.jl | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/test/runtests.jl b/test/runtests.jl index c8b78c5..9545091 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,18 +1,19 @@ using RidgeRegression using Test using DataFrames -using CSV +using LinearAlgebra @testset "RidgeRegression.jl" begin - @testset "Dataset tests" begin + @testset "Dataset Tests" begin include("dataset_tests.jl") end - @testset "Encoding tests" begin + @testset "One-Hot Encoding Tests" begin include("encoding_tests.jl") end - @testset "Load CSV dataset tests" begin + @testset "Load CSV Dataset Tests" begin include("load_csv_dataset_tests.jl") end + end From 121fbc01b2bd5849657b35815e369f95532b3791 Mon Sep 17 00:00:00 2001 From: Eton Tackett Date: Mon, 6 Apr 2026 17:34:08 -0500 Subject: [PATCH 07/21] Design added and tests --- docs/src/design.md | 93 ++++++++++++++++++++++++++++++++++ test/dataset_tests.jl | 4 +- test/encoding_tests.jl | 6 +-- test/load_csv_dataset_tests.jl | 4 +- 4 files changed, 100 insertions(+), 7 deletions(-) create mode 100644 docs/src/design.md diff --git a/docs/src/design.md b/docs/src/design.md new file mode 100644 index 0000000..794fece --- /dev/null +++ b/docs/src/design.md @@ -0,0 +1,93 @@ +# Motivation and Background +Many modern science problems involve regression problems with extremely large numbers of predictors. Genome-wide association studies (GWAS), for example, try to identify genetic variants associated with a disease phenotype using hundreds of thousands or millions of genomic features. In such settings, traditional least squares methods fail because noise and ill-conditioning. Penalized Least Squares (PLS) extends ordinary least squares (OLS) regression by adding a penalty term to shrink parameter estimates. Ridge regression, an approach within PLS, adds a regularization term, producing a regularized estimator. + +Mathematically, ridge regression estimates the regression coefficients by solving the penalized least squares problem +${ +\hat{\boldsymbol{\beta}} = +\arg\min_{\boldsymbol{\beta}} +\left( +\| \mathbf{y} - X\boldsymbol{\beta} \|^2 ++ +\lambda \| \boldsymbol{\beta} \|^2 +\right)} +$ +where $\lambda > 0$ is a regularization parameter that controls the strength of the penalty. + +The purpose of ridge regression is to stabilize regression estimates where the predictors are highly correlated or the design matrix $X$ is almost singular. Ridge regression shrinks the estimated coefficient vector in a way such that the coefficient estimates minimize the sum of squared residuals subject to a constraint on the $\ell_2$ norm of the coefficient vector, $\|\boldsymbol{\beta}\|^2 \leq t$, which shrinks the least squares estimates toward the origin. This reduces the variance of the coefficient estimates and mitigates the effects of multicollinearity. + +There are many numerical algorithms available to compute ridge regression estimates including direct methods, Krylov subspace methods, gradient-based optimization, coordinate descent, and stochastic gradient descent. These algorithms differ in their computational costs and numerical stability. + +The goal of this experiment is to investigate the performance of these algorithms when we vary the structure and scale of the regression problem. To do this, we consider the linear model $\mathbf{y} = X\boldsymbol{\beta} + \boldsymbol{\varepsilon}$ where the matrix ${X}$ may be constructed with varying dimensions, sparsity patterns, and conditioning properties. +# Questions +The primary goal of this experiment is to compare numerical algorithms for computing ridge regression estimates under various conditions. In particular, we aim to address the following questions: + +1. How does the performance of ridge regression algorithms change as the structural and numerical properties of the regression problem vary? + +2. Which ridge regression algorithm provides the best balance between numerical stability and computational cost across these problem regimes? + +# Experimental Units +The experimental units are the datasets under fixed penalty weights. For each experimental unit, all treatments will be applied to the dataset. This will be done so that differences in performance can be attributed to the algorithms themselves rather than the data. Each dataset will contain a matrix ${X}$, a response vector $\mathbf{y}$, and a regularization parameter ${\lambda}$ for some specific ${\lambda}$. + +Blocks are defined by combinations of the experimental blocking factors, including dimensional regime, matrix sparsity, and ridge penalty magnitude. Each block represents datasets with similar structural properties. Within each block, multiple datasets will be generated, and each dataset forms an experimental unit. For every experimental unit all treatments are applied. + +Datasets will be grouped according to their dimensional regime, characterized as $p \ll n$, p ≈ n, and $p \gg n$. These regimes correspond to fundamentally different geometric properties of the design matrix, including rank behavior, conditioning, and the stability of the normal equations. + +In addition to dimensional block, the strength of the ridge penalty will be incorporated as a secondary blocking factor. The ridge estimator is $\hat{\beta_R} = (X^\top X + \lambda I)^{-1}X^\top y$. The matrix conditioning number is defined as $\kappa(A) = \frac{\sigma_{\max}(A)}{\sigma_{\min}(A)}$. In the context of ridge regression, the regularization parameter ${\lambda}$, can impact the conditioning number. Let $X = U\Sigma V^\top$ be the SVD of $X$, with singular values $\sigma_1,\dots,\sigma_p$. + +Then +```math +X^\top X = V \Sigma^\top \Sigma V^\top += V \,\mathrm{diag}(\sigma_1^2,\dots,\sigma_p^2)\, V^\top . +``` + +Adding the ridge term gives + +```math +X^\top X + \lambda I += +V \,\mathrm{diag}(\sigma_1^2+\lambda,\dots,\sigma_p^2+\lambda)\, V^\top . +``` + +```math +\kappa_2(X^\top X+\lambda I) += +\frac{\sigma_{\max}^2+\lambda}{\sigma_{\min}^2+\lambda}. +``` + +Because the performance of numerical algorithms is strongly influenced by the conditioning of the system they solve, the ridge penalty effectively creates regression problems with different numerical difficulty. This provides a way to assess how algorithm performance, convergence behavior, and computational cost depend on the numerical stability of the problem. In this experiment, the magnitude of $\lambda$ is selected relative to the smallest and largest singular values of $X$. A weak regularization regime corresponds to $\lambda \approx \sigma_{\min}^2$, where the ridge penalty begins to influence the smallest singular directions but the system remains moderately ill-conditioned. A moderate regularization regime corresponds to $\lambda \approx \sigma_{\min}\sigma_{\max}$, which substantially improves the conditioning of the problem by increasing the smallest eigenvalues of $X^\top X + \lambda I$. Finally, a strong regularization regime corresponds to $\lambda \approx \sigma_{\max}^2$, where the ridge penalty dominates the spectral scale of the problem and produces a well-conditioned system. + +Another blocking factor that will be considered is how sparse or dense the matrix $X$ is. Many algorithms behave differently depending on whether the matrix is sparse or dense. In ridge regression, there are many operations involving $X$ including matrix-matrix products and matrix-vector products. A dense matrix leads to high computational cost whereas a sparse matrix we can significantly reduce the cost. As such, different algorithms may perform better depending on the sparsity structure of X, making matrix sparsity a relevant blocking factor when comparing algorithm behavior and computational efficiency. + +The total number of block combinations is determined by the product of the number of levels in each blocking factor, denoted b. For example, if the experiment includes three dimensional regimes, two sparsity levels, and two regularization strengths, then there are $3 * 2 * 2 = 12$ block combinations. We will also denote r to be the number of replicated datasets in each block. Here, we mean the number datasets within a block. The total number of experimental units is then ${b * r}$. + +| Blocking System | Factor | Blocks | +|:----------------|:-------|:-------| +| Dataset | Dimensional regime | $(p \ll n)$, $(p \approx n)$, $(p \gg n)$| +| Ridge Penalty | Magnitude of ${\lambda}$ relative to the spectral scale of $X^\top X$ | Weak ($\lambda \approx \sigma_{\min}^2$), Moderate ($\lambda \approx \sigma_{\min}\sigma_{\max}$), Strong ($\lambda \approx \sigma_{\max}^2$), where $\sigma_{\min}$ and $\sigma_{\max}$ denote the smallest and largest singular values of $X$. | +| Matrix Sparsity| Density of non-zero values in $X$ | Sparse (< 10% non-zero), Moderate (10%-50% non-zero), Dense (> 50% non-zero)| +# Treatments + +The treatments are the ridge regression solution methods: + +- Gradient-based optimization +- Stochastic gradient descent +- Direct Methods + - Golub Kahan Bidiagonalization + + Since each experimental unit will recieves all t treatments, the total number of algorithm runs in the experiment is ${t * b * r}$. For this experiment, ${t=3}$. To ensure fair comparison between algorithms, each treatment will be applied under a fixed time constraint. Each algorithm will be run for a maximum of two hours per experimental unit. +# Observational Units and Measurements + +The observational units are each algorithm-dataset pair. For each combination we will observe the following + +| Column Name | Data Type | Description | +|:---|:---|:---| +| `dataset_id` | Positive Integer | Identifier for the generated dataset (experimental unit). | +| `dimensional_regime` | String | Relationship between predictors and observations: `p << n`, `p ≈ n`, or `p >> n`. | +| `sparsity_level` | String | Density of the matrix `X`: `Sparse`, `Moderate`, or `Dense`. | +| `lambda_level` | String | Relative magnitude of the ridge penalty parameter `λ`: `Weak`, `Moderate`, or `Strong`. | +| `algorithm` | String | Ridge regression solution method used: `GradientDescent`, `SGD`, or `DirectMethod`. | +| `runtime_seconds` | Positive Floating-point | Time required for the algorithm to compute a solution. | +| `iterations` | Positive Integer | Number of iterations performed by the algorithm (`NA` for direct methods). | + + +The collected measurements will be written to a CSV file. Each row in the file corresponds to a single algorithm–dataset pair, which forms the observational unit of the experiment. The columns represent the recorded measurements. After the experiment, the resulting CSV file should contain ${Algorithms∗Datasets}$ number of rows and each row will contain exactly seven columns. \ No newline at end of file diff --git a/test/dataset_tests.jl b/test/dataset_tests.jl index a0176a9..a2c035c 100644 --- a/test/dataset_tests.jl +++ b/test/dataset_tests.jl @@ -1,4 +1,4 @@ -@testset "Testset 1" begin +@testset "Dataset constructor stores fields correctly" begin X = [1 2; 3 4] y = [10, 20] d = Dataset("toy", X, y) @@ -12,7 +12,7 @@ @test 20.0 == d.y[2] end -@testset "Testset 2" begin +@testset "Dataset constructor throws error for mismatched dimensions" begin X = [1 2; 3 4] @test_throws ArgumentError Dataset("bad", X, [1, 2, 3]) diff --git a/test/encoding_tests.jl b/test/encoding_tests.jl index ca67e61..7a2d3e5 100644 --- a/test/encoding_tests.jl +++ b/test/encoding_tests.jl @@ -1,4 +1,4 @@ -@testset "Testset 1" begin +@testset "one_hot_encode encodes specified categorical columns and keeps numeric columns" begin df = DataFrame( A = ["red", "blue", "red", "green"], B = [1, 2, 3, 4], @@ -14,7 +14,7 @@ @test all(vec(sum(X[:, 4:5]; dims=2)) .<= 1) end -@testset "Testset 2" begin +@testset "one_hot_encode throws error for invalid column specifications" begin df = DataFrame( A = ["red", "blue", "red", "green"], B = [1, 2, 3, 4], @@ -24,7 +24,7 @@ end @test_throws ArgumentError one_hot_encode(df; cols_to_encode=[:A], drop_first=true) end -@testset "Testset 3" begin +@testset "one_hot_encode supports integer-coded categorical columns when specified" begin df = DataFrame( group = [1, 2, 1, 3], x = [10.0, 20.0, 30.0, 40.0] diff --git a/test/load_csv_dataset_tests.jl b/test/load_csv_dataset_tests.jl index 67f41ed..bd7ddd7 100644 --- a/test/load_csv_dataset_tests.jl +++ b/test/load_csv_dataset_tests.jl @@ -1,4 +1,4 @@ -@testset "Testset 1" begin +@testset "load_csv_dataset drops missing rows and uses target column" begin tmp = tempname() * ".csv" df = DataFrame( @@ -18,7 +18,7 @@ @test (3, 2) == size(d.X) end -@testset "Testset 2" begin +@testset "load_csv_dataset drops missing rows and uses target column by index" begin tmp = tempname() * ".csv" df = DataFrame( From 049bacc79fa536ab8d0575506a5cd13e499a1ee8 Mon Sep 17 00:00:00 2001 From: Eton Tackett Date: Mon, 6 Apr 2026 17:42:20 -0500 Subject: [PATCH 08/21] Fixing project.toml --- test/Project.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/test/Project.toml b/test/Project.toml index b8a3f94..e6f00c7 100644 --- a/test/Project.toml +++ b/test/Project.toml @@ -1,7 +1,6 @@ [deps] CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b" DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" -RidgeRegression = "739161c8-60e1-4c49-8f89-ff30998444b1" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" [compat] From fa3a24a4a0c450d7e0bcc17f7c3329231cd4b24d Mon Sep 17 00:00:00 2001 From: Eton Tackett Date: Mon, 6 Apr 2026 17:51:04 -0500 Subject: [PATCH 09/21] fixing root project.toml --- Project.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/Project.toml b/Project.toml index 9c27dee..427e8aa 100644 --- a/Project.toml +++ b/Project.toml @@ -7,7 +7,6 @@ authors = ["Eton Tackett ", "Vivak Patel Date: Mon, 6 Apr 2026 17:58:13 -0500 Subject: [PATCH 10/21] Adding Linear Algebra --- Project.toml | 1 + src/RidgeRegression.jl | 1 + test/Project.toml | 1 + 3 files changed, 3 insertions(+) diff --git a/Project.toml b/Project.toml index 427e8aa..b49deec 100644 --- a/Project.toml +++ b/Project.toml @@ -7,6 +7,7 @@ authors = ["Eton Tackett ", "Vivak Patel Date: Mon, 6 Apr 2026 18:01:40 -0500 Subject: [PATCH 11/21] Adding csv --- test/runtests.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/test/runtests.jl b/test/runtests.jl index 9545091..8d51d79 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -2,6 +2,7 @@ using RidgeRegression using Test using DataFrames using LinearAlgebra +using CSV @testset "RidgeRegression.jl" begin @testset "Dataset Tests" begin From 715e1d0943d37a54c202f39bc29947abe4a3e84a Mon Sep 17 00:00:00 2001 From: Eton T <154644462+etontackett@users.noreply.github.com> Date: Mon, 22 Jun 2026 17:51:35 -0500 Subject: [PATCH 12/21] Apply suggestion from @vp314 Co-authored-by: Vivak Patel --- src/dataset.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dataset.jl b/src/dataset.jl index 16b3246..09d1bf3 100644 --- a/src/dataset.jl +++ b/src/dataset.jl @@ -61,7 +61,7 @@ One-hot encode categorical (string-like) features in `Xdf`. each categorical feature to avoid multicollinearity. # Returns -- `Matrix{Float64}`: A numeric matrix containing the encoded feature. +- `::Matrix{Float64}`: A numeric matrix containing the encoded feature. """ function one_hot_encode(Xdf::DataFrame; cols_to_encode, drop_first::Bool = true)::Matrix{Float64} n = nrow(Xdf) From ae701b8ba28118eab078b9ab451dc8061a1beaab Mon Sep 17 00:00:00 2001 From: Eton Tackett Date: Tue, 30 Jun 2026 11:38:31 -0500 Subject: [PATCH 13/21] 6/30 Changes --- src/RidgeRegression.jl | 2 +- src/{dataset.jl => units.jl} | 0 test/runtests.jl | 6 +++--- test/{dataset_tests.jl => src/units/units_dataset_tests.jl} | 0 .../units/units_encoding_tests.jl} | 0 .../units/units_load_csv_dataset_tests.jl} | 0 6 files changed, 4 insertions(+), 4 deletions(-) rename src/{dataset.jl => units.jl} (100%) rename test/{dataset_tests.jl => src/units/units_dataset_tests.jl} (100%) rename test/{encoding_tests.jl => src/units/units_encoding_tests.jl} (100%) rename test/{load_csv_dataset_tests.jl => src/units/units_load_csv_dataset_tests.jl} (100%) diff --git a/src/RidgeRegression.jl b/src/RidgeRegression.jl index 509d7c1..e6d4772 100644 --- a/src/RidgeRegression.jl +++ b/src/RidgeRegression.jl @@ -5,7 +5,7 @@ using DataFrames using Downloads using LinearAlgebra -include("dataset.jl") +include("units.jl") export Dataset, load_csv_dataset, one_hot_encode diff --git a/src/dataset.jl b/src/units.jl similarity index 100% rename from src/dataset.jl rename to src/units.jl diff --git a/test/runtests.jl b/test/runtests.jl index 8d51d79..11b4788 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -6,15 +6,15 @@ using CSV @testset "RidgeRegression.jl" begin @testset "Dataset Tests" begin - include("dataset_tests.jl") + include("src/units/units_dataset_tests.jl") end @testset "One-Hot Encoding Tests" begin - include("encoding_tests.jl") + include("src/units/units_encoding_tests.jl") end @testset "Load CSV Dataset Tests" begin - include("load_csv_dataset_tests.jl") + include("src/units/units_load_csv_dataset_tests.jl") end end diff --git a/test/dataset_tests.jl b/test/src/units/units_dataset_tests.jl similarity index 100% rename from test/dataset_tests.jl rename to test/src/units/units_dataset_tests.jl diff --git a/test/encoding_tests.jl b/test/src/units/units_encoding_tests.jl similarity index 100% rename from test/encoding_tests.jl rename to test/src/units/units_encoding_tests.jl diff --git a/test/load_csv_dataset_tests.jl b/test/src/units/units_load_csv_dataset_tests.jl similarity index 100% rename from test/load_csv_dataset_tests.jl rename to test/src/units/units_load_csv_dataset_tests.jl From 506458f4d998ca4ecbbc1dbdccd8e79c613d0678 Mon Sep 17 00:00:00 2001 From: Eton Tackett Date: Fri, 3 Jul 2026 11:06:30 -0500 Subject: [PATCH 14/21] Adding ones column --- src/units.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/src/units.jl b/src/units.jl index 09d1bf3..6665e38 100644 --- a/src/units.jl +++ b/src/units.jl @@ -66,6 +66,7 @@ One-hot encode categorical (string-like) features in `Xdf`. function one_hot_encode(Xdf::DataFrame; cols_to_encode, drop_first::Bool = true)::Matrix{Float64} n = nrow(Xdf) cols = Vector{Vector{Float64}}() + push!(cols, ones(Float64, n)) #Add a column of ones for the intercept term in the design matrix. encode_names = Set(c isa Int ? Symbol(names(Xdf)[c]) : Symbol(c) for c in cols_to_encode) From 45eb36c271ae195ed1cac37424094ca7c4544fcd Mon Sep 17 00:00:00 2001 From: Eton Tackett Date: Mon, 6 Jul 2026 14:05:38 -0500 Subject: [PATCH 15/21] Sync design with main --- docs/src/design.md | 94 ---------------------------------------------- 1 file changed, 94 deletions(-) delete mode 100644 docs/src/design.md diff --git a/docs/src/design.md b/docs/src/design.md deleted file mode 100644 index 225b92a..0000000 --- a/docs/src/design.md +++ /dev/null @@ -1,94 +0,0 @@ -# Motivation and Background -Many modern science problems involve regression problems with extremely large numbers of predictors. Genome-wide association studies (GWAS), for example, try to identify genetic variants associated with a disease phenotype using hundreds of thousands or millions of genomic features. In such settings, traditional least squares methods fail because noise and ill-conditioning. Penalized Least Squares (PLS) extends ordinary least squares (OLS) regression by adding a penalty term to shrink parameter estimates. Ridge regression, an approach within PLS, adds a regularization term, producing a regularized estimator. - -Mathematically, ridge regression estimates the regression coefficients by solving the penalized least squares problem -${ -\hat{\boldsymbol{\beta}} = -\arg\min_{\boldsymbol{\beta}} -\left( -\| \mathbf{y} - X\boldsymbol{\beta} \|^2 -+ -\lambda \| \boldsymbol{\beta} \|^2 -\right)} -$ -where $\lambda > 0$ is a regularization parameter that controls the strength of the penalty. - -The purpose of ridge regression is to stabilize regression estimates where the predictors are highly correlated or the design matrix $X$ is almost singular. Ridge regression shrinks the estimated coefficient vector in a way such that the coefficient estimates minimize the sum of squared residuals subject to a constraint on the $\ell_2$ norm of the coefficient vector, $\|\boldsymbol{\beta}\|^2 \leq t$, which shrinks the least squares estimates toward the origin. This reduces the variance of the coefficient estimates and mitigates the effects of multicollinearity. - -There are many numerical algorithms available to compute ridge regression estimates including direct methods, Krylov subspace methods, gradient-based optimization, coordinate descent, and stochastic gradient descent. These algorithms differ in their computational costs and numerical stability. - -The goal of this experiment is to investigate the performance of these algorithms when we vary the structure and scale of the regression problem. To do this, we consider the linear model $\mathbf{y} = X\boldsymbol{\beta} + \boldsymbol{\varepsilon}$ where the matrix ${X}$ may be constructed with varying dimensions, sparsity patterns, and conditioning properties. -# Questions -The primary goal of this experiment is to compare numerical algorithms for computing ridge regression estimates under various conditions. In particular, we aim to address the following questions: - -1. How does the performance of ridge regression algorithms change as the structural and numerical properties of the regression problem vary? - -2. Which ridge regression algorithm provides the best balance between numerical stability and computational cost across these problem regimes? - -# Experimental Units -The experimental units are the datasets under fixed penalty weights. For each experimental unit, all treatments will be applied to the dataset. This will be done so that differences in performance can be attributed to the algorithms themselves rather than the data. Each dataset will contain a matrix ${X}$, a response vector $\mathbf{y}$, and a regularization parameter ${\lambda}$ for some specific ${\lambda}$. - -Blocks are defined by combinations of the experimental blocking factors, including dimensional regime, matrix sparsity, and ridge penalty magnitude. Each block represents datasets with similar structural properties. Within each block, multiple datasets will be generated, and each dataset forms an experimental unit. For every experimental unit all treatments are applied. - -Datasets will be grouped according to their dimensional regime, characterized as $p \ll n$, p ≈ n, and $p \gg n$. These regimes correspond to fundamentally different geometric properties of the design matrix, including rank behavior, conditioning, and the stability of the normal equations. - -Datasets will be generated either fully artificially or semi-artificially. In the artifical setting, both the design matrix (X) and the response vector (y) are generated. In the semi-artifical setting, the design matrix is fixed from a real dataset (e.g. GWAS) and the response vector is generated conditional on X. This is done so that we can compute solution error. - -Then -```math -X^\top X = V \Sigma^\top \Sigma V^\top -= V \,\mathrm{diag}(\sigma_1^2,\dots,\sigma_p^2)\, V^\top . -``` - -The regularization parameter is often selected using one of the following strategies, -depending on application domain: - -```math -X^\top X + \lambda I -= -V \,\mathrm{diag}(\sigma_1^2+\lambda,\dots,\sigma_p^2+\lambda)\, V^\top . -``` - -```math -\kappa_2(X^\top X+\lambda I) -= -\frac{\sigma_{\max}^2+\lambda}{\sigma_{\min}^2+\lambda}. -``` - -Because the performance of numerical algorithms is strongly influenced by the conditioning of the system they solve, the ridge penalty effectively creates regression problems with different numerical difficulty. This provides a way to assess how algorithm performance, convergence behavior, and computational cost depend on the numerical stability of the problem. In this experiment, the magnitude of $\lambda$ is selected relative to the smallest and largest singular values of $X$. A weak regularization regime corresponds to $\lambda \approx \sigma_{\min}^2$, where the ridge penalty begins to influence the smallest singular directions but the system remains moderately ill-conditioned. A moderate regularization regime corresponds to $\lambda \approx \sigma_{\min}\sigma_{\max}$, which substantially improves the conditioning of the problem by increasing the smallest eigenvalues of $X^\top X + \lambda I$. Finally, a strong regularization regime corresponds to $\lambda \approx \sigma_{\max}^2$, where the ridge penalty dominates the spectral scale of the problem and produces a well-conditioned system. - -Another blocking factor that will be considered is how sparse or dense the matrix $X$ is. Many algorithms behave differently depending on whether the matrix is sparse or dense. In ridge regression, there are many operations involving $X$ including matrix-matrix products and matrix-vector products. A dense matrix leads to high computational cost whereas a sparse matrix we can significantly reduce the cost. As such, different algorithms may perform better depending on the sparsity structure of X, making matrix sparsity a relevant blocking factor when comparing algorithm behavior and computational efficiency. - -The total number of block combinations is determined by the product of the number of levels in each blocking factor, denoted b. For example, if the experiment includes three dimensional regimes, two sparsity levels, and two regularization strengths, then there are $3 * 2 * 2 = 12$ block combinations. We will also denote r to be the number of replicated datasets in each block. Here, we mean the number datasets within a block. The total number of experimental units is then ${b * r}$. - -| Blocking System | Factor | Blocks | -|:----------------|:-------|:-------| -| Dataset | Dimensional regime | $(p \ll n)$, $(p \approx n)$, $(p \gg n)$| -| Ridge Penalty | Magnitude of ${\lambda}$ relative to the spectral scale of $X^\top X$ | Weak ($\lambda \approx \sigma_{\min}^2$), Moderate ($\lambda \approx \sigma_{\min}\sigma_{\max}$), Strong ($\lambda \approx \sigma_{\max}^2$), where $\sigma_{\min}$ and $\sigma_{\max}$ denote the smallest and largest singular values of $X$. | -| Matrix Sparsity| Density of non-zero values in $X$ | Sparse (< 10% non-zero), Moderate (10%-50% non-zero), Dense (> 50% non-zero)| -# Treatments - -The treatments are the ridge regression solution methods: - -- Gradient-based optimization -- Stochastic gradient descent -- Direct Methods - - Golub Kahan Bidiagonalization - - Since each experimental unit will recieves all t treatments, the total number of algorithm runs in the experiment is ${t * b * r}$. For this experiment, ${t=3}$. To ensure fair comparison between algorithms, each treatment will be applied under a fixed time constraint. Each algorithm will be run for a maximum of two hours per experimental unit. -# Observational Units and Measurements - -The observational units are each algorithm-dataset pair. For each combination we will observe the following - -| Column Name | Data Type | Description | -|:---|:---|:---| -| `dataset_id` | Positive Integer | Identifier for the generated dataset (experimental unit). | -| `dimensional_regime` | String | Relationship between predictors and observations: `p << n`, `p ≈ n`, or `p >> n`. | -| `sparsity_level` | String | Density of the matrix `X`: `Sparse`, `Moderate`, or `Dense`. | -| `lambda_level` | String | Relative magnitude of the ridge penalty parameter `λ`: `Weak`, `Moderate`, or `Strong`. | -| `algorithm` | String | Ridge regression solution method used: `GradientDescent`, `SGD`, or `DirectMethod`. | -| `runtime_seconds` | Positive Floating-point | Time required for the algorithm to compute a solution. | -| `iterations` | Positive Integer | Number of iterations performed by the algorithm (`NA` for direct methods). | - - -The collected measurements will be written to a CSV file. Each row in the file corresponds to a single algorithm–dataset pair, which forms the observational unit of the experiment. The columns represent the recorded measurements. After the experiment, the resulting CSV file should contain ${Algorithms∗Datasets}$ number of rows and each row will contain exactly seven columns. From 8132054af676ba253c3b22a1f70ac1830a63c081 Mon Sep 17 00:00:00 2001 From: Eton Tackett Date: Mon, 6 Jul 2026 14:26:52 -0500 Subject: [PATCH 16/21] units.jl changes --- src/units.jl | 50 +++++++++++++++++++++++++++++--------------------- 1 file changed, 29 insertions(+), 21 deletions(-) diff --git a/src/units.jl b/src/units.jl index 6665e38..94dbad6 100644 --- a/src/units.jl +++ b/src/units.jl @@ -1,49 +1,57 @@ """ - Dataset <: ExperimentalUnit + Unit{TX<:AbstractMatrix, TY<:AbstractVector, Tλ<:Real} -A dataset for Ridge Regression experiements. +An experimental unit for ridge regression experiments. # Description -A `Dataset` object stores the design matrix ``X`` and response vector ``y`` -for a regression problem. These datasets serve as the experimental units for ridge regression experiments, allowing us to evaluate the performance of ridge regression models on various datasets. +A `Unit` object stores the design matrix `X`, response vector `y`, +regularization parameter `λ`, and dimensions `n` and `p` +for a ridge regression problem. # Fields -- `name::String`: Name of dataset +- `name::String`: Name of the unit - `X::TX`: Matrix of variables/features - `y::TY`: Target vector +- `λ::Tλ`: Regularization parameter for ridge regression +- `n::Int`: Number of rows +- `p::Int`: Number of columns # Constructor - Dataset(name::String, X::AbstractMatrix, y::AbstractVector) + Unit(name::String, X::AbstractMatrix, y::AbstractVector, λ::Real) ## Arguments -- `name::String`: Name of dataset -- `X::TX`: Matrix of variables/features -- `y::TY`: Target vector +- `name::String`: Name of the unit +- `X::AbstractMatrix`: Matrix of variables/features +- `y::AbstractVector`: Target vector +- `λ::Real`: Regularization parameter for ridge regression ## Returns -- A `Dataset` object containing the numeric design matrix and response vector. +- A `Unit` object containing the design matrix, response vector, regularization parameter, and dimensions. ## Throws -- `ArgumentError`: If rows in `X` does not equal length of `y`. - -!!! note - `Dataset` objects are used as experimental units when evaluating - ridge regression algorithms. The parametric design allows both dense - and sparse matrices to be stored without forcing conversion to a - dense `Matrix{Float64}`. +- `ArgumentError`: If rows in `X` do not equal length of `y`. """ -struct Dataset{TX<:AbstractMatrix, TY<:AbstractVector} +struct Unit{TX<:AbstractMatrix, TY<:AbstractVector, Tλ<:Real} name::String X::TX y::TY - - function Dataset(name::String, X::TX, y::TY) where {TX<:AbstractMatrix, TY<:AbstractVector} + λ::Tλ + n::Int + p::Int + + function Unit(name::String, X::TX, y::TY, λ::Tλ) where { + TX<:AbstractMatrix, + TY<:AbstractVector, + Tλ<:Real + } size(X, 1) == length(y) || throw(ArgumentError("X and y must have same number of rows")) - new{TX, TY}(name, X, y) + n, p = size(X) + + new{TX, TY, Tλ}(name, X, y, λ, n, p) end end From eb5e7965961167d2bfa522942c9790c8ca86faca Mon Sep 17 00:00:00 2001 From: Eton Tackett Date: Mon, 6 Jul 2026 14:27:51 -0500 Subject: [PATCH 17/21] units.jl changes --- docs/src/design.md | 182 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 182 insertions(+) create mode 100644 docs/src/design.md diff --git a/docs/src/design.md b/docs/src/design.md new file mode 100644 index 0000000..db8541e --- /dev/null +++ b/docs/src/design.md @@ -0,0 +1,182 @@ +# Motivation and Background +Many modern science problems involve regression problems with extremely large numbers of +predictors. +[Genome-wide association studies (GWAS)](https://doi.org/10.1371/journal.pone.0245376), +for example, try to identify genetic +variants associated with a disease phenotype using hundreds of thousands or millions of +genomic features. In such settings, traditional least squares methods fail. +Penalized Least Squares (PLS) extends ordinary least squares (OLS) +regression by adding a penalty term to shrink parameter estimates. Ridge regression, an +approach within PLS, adds a regularization term, producing a regularized estimator. + +Mathematically, ridge regression estimates the regression coefficients by solving the +penalized least squares problem +```math +\hat{\boldsymbol{\beta}} = +\arg\min_{\boldsymbol{\beta}} +\left( +\| \mathbf{y} - X\boldsymbol{\beta} \|_2^2 ++ +\lambda \| \boldsymbol{\beta} \|_2^2 +\right) +``` +where $\lambda > 0$ is a regularization parameter that controls the strength of the penalty. + +The purpose of ridge regression is to stabilize regression estimates when the predictors are +highly correlated or the design matrix $X$ is nearly singular. Ridge regression modifies the +least squares objective by adding a penalty on the squared $\ell_2$-norm of the coefficient +vector. The estimator is obtained by minimizing a penalized least squares objective in which +large coefficient values are discouraged through the penalty term $\lambda +\|\boldsymbol{\beta}\|_2^2$. This penalty shrinks the estimated coefficients toward the +origin, which reduces the variance of the estimator and mitigates the effects of +multicollinearity. + +There are many numerical algorithms available to compute ridge regression estimates +including direct methods, Krylov subspace methods, gradient-based optimization, coordinate +descent, and stochastic gradient descent. These algorithms differ in their computational +costs and numerical stability. + +The goal of this experiment is to investigate the performance of these algorithms when we +vary the structure and scale of the regression problem. To do this, we consider the linear +model $\mathbf{y} = X\boldsymbol{\beta} + \boldsymbol{\varepsilon}$ where the matrix ${X}$ +may be constructed with varying dimensions, sparsity patterns, and conditioning properties. +# Questions +The primary goal of this experiment is to compare numerical algorithms for computing ridge +regression estimates under various conditions. In particular, we aim to address the +following questions: + +1. How does the performance of ridge regression algorithms change as the structural + and numerical properties of the regression problem vary? + +2. Which ridge regression algorithm provides the best balance between numerical + stability and computational cost across these problem regimes? + +# Experimental Units +An experimental unit is defined by a triplet: a design matrix, $X$; a response vector, $y$; +and a non-negative regularization parameter $\lambda$. + +Datasets will be generated either fully artificially or semi-artificially. In the artifical setting, both the design matrix (X) and the response vector (y) are generated. In the semi-artifical setting, the design matrix is fixed from a real dataset (e.g. GWAS) and the response vector is generated conditional on X. This is done so that we can compute solution error. + +Experimental units are selected based on factors we believe will potentially impact +the performance of a specific algorithm. +These factors, or blocks, correspond to: +- The dimensional regime of $X \in \mathbb{R}^{n \times p}$: + $p \ll n$, $p ≈ n$, and $p \gg n$. +- The magnitude of the regularization parameter, $\lambda$, as described below. + +Matrix sparsity is a factor that can influence algorithm performance. However, we will not consider it in this experiment because none of the algorithms under consideration are implemented in a manner that exploits sparse matrix structure. + +The regularization parameter is often selected using one of the following strategies, +depending on application domain: + +- [Generalized Cross Validation](https://doi.org/10.1080/00401706.1979.10489751) +- [The L-Curve Method](https://doi.org/10.1137/1034115) +- [Information Criteria](https://doi.org/10.1002/wics.1607) +- [Morozov's Discrepancy Principle](https://doi.org/10.1088/0266-5611/26/2/025001) + +Excluding Morozov's discrepancy principle, the remaining methods require +computing $\hat{\beta}_\lambda = (X^\top X + \lambda I)^{-1}X^\top y$ or +$\Vert X\hat{\beta}_\lambda - y \Vert_2$ over a grid of values for $\lambda$. +Given our assumption that problem conditioning will impact algorithm behavior, +we will choose this grid to modify the problem's condition number systematically. + +Recall, the ridge estimator's closed form is given by solving +```math +\min_{\beta} \left\Vert \begin{bmatrix} X \\ \lambda I \end{bmatrix}\beta - +y \right\Vert_2^2, +``` +or (equivalently, from a mathematical perspective) solving +$(X^\top X + \lambda I)\hat{\beta}_\lambda = X^\top y$. +Let $\sigma_{\max}$ denote the largest singular value +of $X$, and let $\sigma_{\min}$ denote the smallest singular value of +$X$ (note, $0$ is allowed). +Then the condition number for the normal systems approach is +```math +\kappa(X^\top X+\lambda I) += +\frac{\sigma_{\max}^2+\lambda}{\sigma_{\min}^2+\lambda}. +``` + +There are two cases we consider. +- Case 1: $\sigma_{\min}=0$. In this case, the least squares problem is ill-posed. We choose + the smallest value of $\lambda$ in the grid to be + $\sigma_{\max}^2 / (10^{12} - 1)$, which ensures + $\kappa(X^\top X + \lambda I) = 10^{12}$. +- Case 2: $\sigma_{\min}>0$. In this case, we choose the smallest value of $\lambda$ in the + grid to be $\sigma_{\max}\sigma_{\min}$, which ensures + $\kappa(X^\top X + \lambda I) = \sigma_{\max}/\sigma_{\min}$. + That is, it sets the regularized normal system's condition number to that of the original + least squares problem. + +In both cases, assuming $\sigma_{\max}^2 > 2 \sigma_{\min}$, we choose the larges value of +$\lambda$ to be $\sigma_{\max}^2 - 2 \sigma_{\min}$. The condition number for the +regularized normal equations is $\kappa(X^\top X + \lambda I)=2$. + +For each experimental unit, a grid of 15 regularization parameters will be constructed between $\lambda _{min}$ and $\lambda _{max}$. The endpoints of the grid are chosen to correspond to the range of condition numbers of the normal equations. Intermediate values will be selected using a log-scale. We will use logarithmic scaling because $\kappa(X^\top X+\lambda I)$ changes rapidly as $\lambda \rightarrow 0$, making linear spacing insufficient for capturing performance. + +Suppose we have m values, $\lambda _1\:,...,\:\lambda _m$ where $\lambda _1=\lambda_{min}$ and $\lambda _m=\lambda _{max}$. Logarithmic scaling tells us we want the ratio between these values to be constant. To ensure this, we consider a geometric series of the form: +```math +\lambda _i=\lambda _{min}\left(r\right)^{i-1}, i = 1, 2, ..., 15 +``` +Our goal here is to derive the common ratio r. Relating $\lambda _{min}$ and $\lambda _{max}$ gives us +```math +\lambda _{max}=\lambda _{min}\left(r\right)^{m-1} +``` +Using algebra to isolate r +```math +\frac{\lambda_{\text{max}}}{\lambda_{\text{min}}} = r^{m-1} +``` +Applying logarithms and rearranging we obtain +```math +r=e^{\frac{log\left(\frac{\lambda _{max}}{\lambda _{min}}\right)}{m-1}}=\left(\frac{\lambda _{max}}{\lambda _{min}}\right)^{\frac{1}{m-1}} +``` +In our case $m=15$ + +The total number of block combinations is determined by the product of the number of levels +in each factor, denoted b. We will also denote r to be the number of replicated +datasets in each block. Here, we mean the number datasets within a block. The total number +of experimental units is then ${b * r}$. + + +| Blocking System | Factor | Blocks | +|:----------------|:-------|:-------| +| Dataset | Dimensional regime | $(p \ll n)$, $(p \approx n)$, $(p \gg n)$| +| Conditioning | Condition number of $X^\top X + \lambda I$ | 15 logartihmically spaced $\lambda$ values so that $\kappa(X^\top X+\lambda I)$ ranges from poorly conditioned to conditioned. | + +# Treatments + +The treatments are the ridge regression solution methods: + +- Gradient-based optimization +- Stochastic gradient descent +- Direct Methods + - Golub Kahan Bidiagonalization + + Since each experimental unit will recieves all t treatments, the total number of algorithm + runs in the experiment is ${t * b * r}$. For this experiment, ${t=3}$. To ensure fair + comparison between algorithms, each treatment will be applied under a fixed wall time + constraint. Each algorithm will be run for a maximum of two hours per experimental unit. +# Observational Units and Measurements + +The observational units are each algorithm-dataset pair. For each combination we will observe the following + +| Column Name | Data Type | Description | +|:---|:---|:---| +| `dataset_id` | Positive Integer | Identifier for the generated dataset (experimental unit). | +| `dimensional_regime` | String | Relationship between predictors and observations: `p << n`, `p ≈ n`, or `p >> n`. | +| `lambda` | Positive Floating-point| Regularization parameter used. | +| `condition_number` | Positive Floating-point| $\kappa(X^\top X+\lambda I)$ corresponding to the selected $\lambda$ | +| `algorithm` | String | Ridge regression solution method used: `GradientDescent`, `SGD`, or `DirectMethod`. | +| `runtime_seconds` | Positive Floating-point | Time required for the algorithm to compute a solution. | +| `iterations` | Positive Integer | Number of iterations performed by the algorithm (`NA` for direct methods). | +| `step_size` | Positive Floating-point | Step size used in gradient descent or SGD (`NA` for direct methods). | +| `batch_size` | Positive Integer | Number of samples used per SGD update (`NA` for direct methods and gradient descent). | +| `number_of_epochs` | Positive Integer | Number epochs per observation (`NA` for direct methods). | + + + +The collected measurements will be written to a CSV file. Each row in the file corresponds +to a single algorithm–dataset pair, which forms the observational unit of the experiment. +The columns represent the recorded measurements. After the experiment, the resulting CSV +file should contain ${Algorithms∗Datasets}$ number of rows and each row will contain exactly +10 columns. \ No newline at end of file From 8ffb1678772b0364cce7553c486441fead55ea2e Mon Sep 17 00:00:00 2001 From: Eton Tackett Date: Mon, 6 Jul 2026 14:40:47 -0500 Subject: [PATCH 18/21] Additional Changes --- src/units.jl | 5 ++++- test/src/units/units_dataset_tests.jl | 14 +++++++++----- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/src/units.jl b/src/units.jl index 94dbad6..0b7b7d5 100644 --- a/src/units.jl +++ b/src/units.jl @@ -70,6 +70,9 @@ One-hot encode categorical (string-like) features in `Xdf`. # Returns - `::Matrix{Float64}`: A numeric matrix containing the encoded feature. + +# Throws +- `ArgumentError`: If a column in `Xdf` is not numeric and not listed in `cols_to_encode`. """ function one_hot_encode(Xdf::DataFrame; cols_to_encode, drop_first::Bool = true)::Matrix{Float64} n = nrow(Xdf) @@ -115,7 +118,7 @@ end """ load_csv_dataset(path_or_url; target_col, name="csv_dataset") -Load a dataset from a CSV file or URL. +Load a dataset from a CSV file or URL and removes rows with missing values. # Arguments - `path_or_url::String`: Local file path or web URL containing CSV data. diff --git a/test/src/units/units_dataset_tests.jl b/test/src/units/units_dataset_tests.jl index a2c035c..8ba8f2c 100644 --- a/test/src/units/units_dataset_tests.jl +++ b/test/src/units/units_dataset_tests.jl @@ -1,19 +1,23 @@ -@testset "Dataset constructor stores fields correctly" begin +@testset "Unit constructor stores fields correctly" begin X = [1 2; 3 4] y = [10, 20] - d = Dataset("toy", X, y) + λ = 0.1 + d = Unit("toy", X, y, λ) @test "toy" == d.name @test X == d.X @test y == d.y + @test λ == d.λ + @test 2 == d.n + @test 2 == d.p @test (2, 2) == size(d.X) @test 2 == length(d.y) @test 1.0 == d.X[1, 1] @test 20.0 == d.y[2] end -@testset "Dataset constructor throws error for mismatched dimensions" begin +@testset "Unit constructor throws error for mismatched dimensions" begin X = [1 2; 3 4] - - @test_throws ArgumentError Dataset("bad", X, [1, 2, 3]) + λ = 0.1 + @test_throws ArgumentError Unit("bad", X, [1, 2, 3], λ) end From bb156beeff91b58cc8ddd923fb647ba0a8264097 Mon Sep 17 00:00:00 2001 From: Eton Tackett Date: Mon, 6 Jul 2026 14:47:40 -0500 Subject: [PATCH 19/21] Testing changes --- src/units.jl | 8 +++++--- test/src/units/units_load_csv_dataset_tests.jl | 14 +++++++++++--- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/src/units.jl b/src/units.jl index 0b7b7d5..5e3f39f 100644 --- a/src/units.jl +++ b/src/units.jl @@ -127,11 +127,13 @@ Load a dataset from a CSV file or URL and removes rows with missing values. - `cols_to_encode=Symbol[]`: Column names or indices in the feature data to one-hot encode. - `target_col`: Column index or column name containing the response variable. - `name::String="csv_dataset"`: Dataset name. +- `λ::Real=1.0`: Regularization parameter for ridge regression. # Returns -- `Dataset`: A dataset containing the encoded feature matrix `X`, response vector `y`, and dataset name. +- `Unit`: A unit containing the encoded feature matrix `X`, response vector `y`, + regularization parameter `λ`, and dimensions `n` and `p`. """ -function load_csv_dataset(path_or_url::String; cols_to_encode=Symbol[], target_col, name::String = "csv_dataset") +function load_csv_dataset(path_or_url::String; cols_to_encode=Symbol[], target_col, name::String = "csv_dataset", λ::Real=1.0) filepath = startswith(path_or_url, "http") ? @@ -152,5 +154,5 @@ function load_csv_dataset(path_or_url::String; cols_to_encode=Symbol[], target_ X = one_hot_encode(Xdf; cols_to_encode=encode_cols, drop_first = true) - return Dataset(name, X, collect(Float64, y)) + return Unit(name, X, collect(Float64, y), λ) end diff --git a/test/src/units/units_load_csv_dataset_tests.jl b/test/src/units/units_load_csv_dataset_tests.jl index bd7ddd7..c6e3e6a 100644 --- a/test/src/units/units_load_csv_dataset_tests.jl +++ b/test/src/units/units_load_csv_dataset_tests.jl @@ -9,9 +9,13 @@ CSV.write(tmp, df) - d = load_csv_dataset(tmp; target_col=:y, cols_to_encode=[:b], name="tmp") + λ = 0.1 + d = load_csv_dataset(tmp; target_col=:y, cols_to_encode=[:b], name="tmp", λ=λ) @test "tmp" == d.name + @test λ == d.λ + @test 3 == d.n + @test 2 == d.p @test 3 == length(d.y) @test 3 == size(d.X, 1) @test [10.0, 20.0, 40.0] == d.y @@ -29,10 +33,14 @@ end CSV.write(tmp, df) - d = load_csv_dataset(tmp; target_col=3, cols_to_encode=[:b], name="tmp2") + λ = 0.5 + d = load_csv_dataset(tmp; target_col=3, cols_to_encode=[:b], name="tmp2", λ=λ) @test "tmp2" == d.name + @test λ == d.λ + @test 3 == d.n + @test 2 == d.p @test [10.0, 20.0, 40.0] == d.y @test 3 == size(d.X, 1) @test (3, 2) == size(d.X) -end +end \ No newline at end of file From 8d5b83e42c85216c00469c223e9248ad4716599c Mon Sep 17 00:00:00 2001 From: Eton Tackett Date: Mon, 6 Jul 2026 14:59:40 -0500 Subject: [PATCH 20/21] Tests Changed --- test/Project.toml | 1 + test/src/units/units_encoding_tests.jl | 20 ++++++++++--------- .../src/units/units_load_csv_dataset_tests.jl | 10 ++++++---- 3 files changed, 18 insertions(+), 13 deletions(-) diff --git a/test/Project.toml b/test/Project.toml index 1432e02..118565d 100644 --- a/test/Project.toml +++ b/test/Project.toml @@ -3,6 +3,7 @@ CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b" DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" +RidgeRegression = "739161c8-60e1-4c49-8f89-ff30998444b1" [compat] CSV = "0.10" diff --git a/test/src/units/units_encoding_tests.jl b/test/src/units/units_encoding_tests.jl index 7a2d3e5..5b57c35 100644 --- a/test/src/units/units_encoding_tests.jl +++ b/test/src/units/units_encoding_tests.jl @@ -7,11 +7,12 @@ X = one_hot_encode(df; cols_to_encode=[:A, :C], drop_first=true) - @test (4, 5) == size(X) - @test [1.0, 2.0, 3.0, 4.0] == X[:, 3] - @test all(x -> x == 0.0 || x == 1.0, X[:, [1, 2, 4, 5]]) - @test all(vec(sum(X[:, 1:2]; dims=2)) .<= 1) - @test all(vec(sum(X[:, 4:5]; dims=2)) .<= 1) + @test (4, 6) == size(X) + @test all(X[:, 1] .== 1.0) + @test [1.0, 2.0, 3.0, 4.0] == X[:, 4] + @test all(x -> x == 0.0 || x == 1.0, X[:, [2, 3, 5, 6]]) + @test all(vec(sum(X[:, 2:3]; dims=2)) .<= 1) + @test all(vec(sum(X[:, 5:6]; dims=2)) .<= 1) end @testset "one_hot_encode throws error for invalid column specifications" begin @@ -32,7 +33,8 @@ end X = one_hot_encode(df; cols_to_encode=[:group], drop_first=true) - @test (4, 3) == size(X) - @test [10.0, 20.0, 30.0, 40.0] == X[:, 3] - @test all(x -> x == 0.0 || x == 1.0, X[:, 1:2]) -end + @test (4, 4) == size(X) + @test all(X[:, 1] .== 1.0) + @test [10.0, 20.0, 30.0, 40.0] == X[:, 4] + @test all(x -> x == 0.0 || x == 1.0, X[:, 2:3]) +end \ No newline at end of file diff --git a/test/src/units/units_load_csv_dataset_tests.jl b/test/src/units/units_load_csv_dataset_tests.jl index c6e3e6a..208a7dd 100644 --- a/test/src/units/units_load_csv_dataset_tests.jl +++ b/test/src/units/units_load_csv_dataset_tests.jl @@ -15,11 +15,12 @@ @test "tmp" == d.name @test λ == d.λ @test 3 == d.n - @test 2 == d.p + @test 3 == d.p @test 3 == length(d.y) @test 3 == size(d.X, 1) + @test all(d.X[:, 1] .== 1.0) @test [10.0, 20.0, 40.0] == d.y - @test (3, 2) == size(d.X) + @test (3, 3) == size(d.X) end @testset "load_csv_dataset drops missing rows and uses target column by index" begin @@ -39,8 +40,9 @@ end @test "tmp2" == d.name @test λ == d.λ @test 3 == d.n - @test 2 == d.p + @test 3 == d.p + @test all(d.X[:, 1] .== 1.0) @test [10.0, 20.0, 40.0] == d.y @test 3 == size(d.X, 1) - @test (3, 2) == size(d.X) + @test (3, 3) == size(d.X) end \ No newline at end of file From 382bb3cb39399974e0328b459df8f055b795a0eb Mon Sep 17 00:00:00 2001 From: Eton Tackett Date: Mon, 6 Jul 2026 15:10:26 -0500 Subject: [PATCH 21/21] Units Changes --- test/src/units/units_dataset_tests.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/src/units/units_dataset_tests.jl b/test/src/units/units_dataset_tests.jl index 8ba8f2c..d2332c1 100644 --- a/test/src/units/units_dataset_tests.jl +++ b/test/src/units/units_dataset_tests.jl @@ -2,7 +2,7 @@ X = [1 2; 3 4] y = [10, 20] λ = 0.1 - d = Unit("toy", X, y, λ) + d = RidgeRegression.Unit("toy", X, y, λ) @test "toy" == d.name @test X == d.X @@ -19,5 +19,5 @@ end @testset "Unit constructor throws error for mismatched dimensions" begin X = [1 2; 3 4] λ = 0.1 - @test_throws ArgumentError Unit("bad", X, [1, 2, 3], λ) + @test_throws ArgumentError RidgeRegression.Unit("bad", X, [1, 2, 3], λ) end