diff --git a/test/test_optim.py b/test/test_optim.py index ee4abc81c6af2..85e55e912f59d 100644 --- a/test/test_optim.py +++ b/test/test_optim.py @@ -260,6 +260,8 @@ def test_sgd(self): self._build_params_dict_single(weight, bias, lr=1e-2), lr=1e-3) ) + with self.assertRaisesRegex(ValueError, "Invalid momentum value: -0.5"): + optim.SGD(None, lr=1e-2, momentum=-0.5) def test_sgd_sparse(self): self._test_rosenbrock_sparse( @@ -300,6 +302,8 @@ def test_sparse_adam(self): lambda params: optim.SparseAdam(params, lr=4e-2), True ) + with self.assertRaisesRegex(ValueError, "Invalid beta parameter at index 0: 1.0"): + optim.SparseAdam(None, lr=1e-2, betas=(1.0, 0.0)) def test_adadelta(self): self._test_rosenbrock( @@ -321,6 +325,8 @@ def test_adadelta(self): lambda weight, bias: optim.Adadelta( self._build_params_dict(weight, bias, rho=0.95)) ) + with self.assertRaisesRegex(ValueError, "Invalid rho value: 1.1"): + optim.Adadelta(None, lr=1e-2, rho=1.1) def test_adagrad(self): self._test_rosenbrock( @@ -343,6 +349,8 @@ def test_adagrad(self): self._build_params_dict(weight, bias, lr=1e-2), lr=1e-1) ) + with self.assertRaisesRegex(ValueError, "Invalid lr_decay value: -0.5"): + optim.Adagrad(None, lr=1e-2, lr_decay=-0.5) def test_adagrad_sparse(self): self._test_rosenbrock_sparse( @@ -370,6 +378,8 @@ def test_adamax(self): self._build_params_dict(weight, bias, lr=1e-2), lr=1e-1) ) + with self.assertRaisesRegex(ValueError, "Invalid beta parameter at index 1: 1.0"): + optim.Adamax(None, lr=1e-2, betas=(0.0, 1.0)) def test_rmsprop(self): self._test_rosenbrock( @@ -392,6 +402,8 @@ def test_rmsprop(self): self._build_params_dict(weight, bias, lr=1e-3), lr=1e-2) ) + with self.assertRaisesRegex(ValueError, "Invalid momentum value: -1.0"): + optim.RMSprop(None, lr=1e-2, momentum=-1.0) def test_asgd(self): self._test_rosenbrock( @@ -414,6 +426,8 @@ def test_asgd(self): self._build_params_dict(weight, bias, lr=1e-2), lr=1e-3, t0=100) ) + with self.assertRaisesRegex(ValueError, "Invalid weight_decay value: -0.5"): + optim.ASGD(None, lr=1e-2, weight_decay=-0.5) def test_rprop(self): self._test_rosenbrock( @@ -436,6 +450,8 @@ def test_rprop(self): self._build_params_dict(weight, bias, lr=1e-2), lr=1e-3) ) + with self.assertRaisesRegex(ValueError, "Invalid eta values: 1.0, 0.5"): + optim.Rprop(None, lr=1e-2, etas=(1.0, 0.5)) def test_lbfgs(self): self._test_rosenbrock( diff --git a/torch/optim/adadelta.py b/torch/optim/adadelta.py index a37febaab5414..f69de33e474aa 100644 --- a/torch/optim/adadelta.py +++ b/torch/optim/adadelta.py @@ -23,6 +23,15 @@ class Adadelta(Optimizer): """ def __init__(self, params, lr=1.0, rho=0.9, eps=1e-6, weight_decay=0): + if not 0.0 <= lr: + raise ValueError("Invalid learning rate: {}".format(lr)) + if not 0.0 <= rho <= 1.0: + raise ValueError("Invalid rho value: {}".format(rho)) + if not 0.0 <= eps: + raise ValueError("Invalid epsilon value: {}".format(eps)) + if not 0.0 <= weight_decay: + raise ValueError("Invalid weight_decay value: {}".format(weight_decay)) + defaults = dict(lr=lr, rho=rho, eps=eps, weight_decay=weight_decay) super(Adadelta, self).__init__(params, defaults) diff --git a/torch/optim/adagrad.py b/torch/optim/adagrad.py index 7c152df1b0506..9f103ab957524 100644 --- a/torch/optim/adagrad.py +++ b/torch/optim/adagrad.py @@ -20,6 +20,13 @@ class Adagrad(Optimizer): """ def __init__(self, params, lr=1e-2, lr_decay=0, weight_decay=0): + if not 0.0 <= lr: + raise ValueError("Invalid learning rate: {}".format(lr)) + if not 0.0 <= lr_decay: + raise ValueError("Invalid lr_decay value: {}".format(lr_decay)) + if not 0.0 <= weight_decay: + raise ValueError("Invalid weight_decay value: {}".format(weight_decay)) + defaults = dict(lr=lr, lr_decay=lr_decay, weight_decay=weight_decay) super(Adagrad, self).__init__(params, defaults) diff --git a/torch/optim/adam.py b/torch/optim/adam.py index a854a2780dcc9..1339579765fe6 100644 --- a/torch/optim/adam.py +++ b/torch/optim/adam.py @@ -28,6 +28,10 @@ class Adam(Optimizer): def __init__(self, params, lr=1e-3, betas=(0.9, 0.999), eps=1e-8, weight_decay=0, amsgrad=False): + if not 0.0 <= lr: + raise ValueError("Invalid learning rate: {}".format(lr)) + if not 0.0 <= eps: + raise ValueError("Invalid epsilon value: {}".format(eps)) if not 0.0 <= betas[0] < 1.0: raise ValueError("Invalid beta parameter at index 0: {}".format(betas[0])) if not 0.0 <= betas[1] < 1.0: diff --git a/torch/optim/adamax.py b/torch/optim/adamax.py index dfec5ac371943..c6c803a2898f0 100644 --- a/torch/optim/adamax.py +++ b/torch/optim/adamax.py @@ -22,6 +22,17 @@ class Adamax(Optimizer): def __init__(self, params, lr=2e-3, betas=(0.9, 0.999), eps=1e-8, weight_decay=0): + if not 0.0 <= lr: + raise ValueError("Invalid learning rate: {}".format(lr)) + if not 0.0 <= eps: + raise ValueError("Invalid epsilon value: {}".format(eps)) + if not 0.0 <= betas[0] < 1.0: + raise ValueError("Invalid beta parameter at index 0: {}".format(betas[0])) + if not 0.0 <= betas[1] < 1.0: + raise ValueError("Invalid beta parameter at index 1: {}".format(betas[1])) + if not 0.0 <= weight_decay: + raise ValueError("Invalid weight_decay value: {}".format(weight_decay)) + defaults = dict(lr=lr, betas=betas, eps=eps, weight_decay=weight_decay) super(Adamax, self).__init__(params, defaults) diff --git a/torch/optim/asgd.py b/torch/optim/asgd.py index f72d1b20c6a62..05c9903bc06a4 100644 --- a/torch/optim/asgd.py +++ b/torch/optim/asgd.py @@ -23,6 +23,11 @@ class ASGD(Optimizer): """ def __init__(self, params, lr=1e-2, lambd=1e-4, alpha=0.75, t0=1e6, weight_decay=0): + if not 0.0 <= lr: + raise ValueError("Invalid learning rate: {}".format(lr)) + if not 0.0 <= weight_decay: + raise ValueError("Invalid weight_decay value: {}".format(weight_decay)) + defaults = dict(lr=lr, lambd=lambd, alpha=alpha, t0=t0, weight_decay=weight_decay) super(ASGD, self).__init__(params, defaults) diff --git a/torch/optim/rmsprop.py b/torch/optim/rmsprop.py index 2606c68492334..cbb28f79ff16d 100644 --- a/torch/optim/rmsprop.py +++ b/torch/optim/rmsprop.py @@ -26,6 +26,17 @@ class RMSprop(Optimizer): """ def __init__(self, params, lr=1e-2, alpha=0.99, eps=1e-8, weight_decay=0, momentum=0, centered=False): + if not 0.0 <= lr: + raise ValueError("Invalid learning rate: {}".format(lr)) + if not 0.0 <= eps: + raise ValueError("Invalid epsilon value: {}".format(eps)) + if not 0.0 <= momentum: + raise ValueError("Invalid momentum value: {}".format(momentum)) + if not 0.0 <= weight_decay: + raise ValueError("Invalid weight_decay value: {}".format(weight_decay)) + if not 0.0 <= alpha: + raise ValueError("Invalid alpha value: {}".format(alpha)) + defaults = dict(lr=lr, momentum=momentum, alpha=alpha, eps=eps, centered=centered, weight_decay=weight_decay) super(RMSprop, self).__init__(params, defaults) diff --git a/torch/optim/rprop.py b/torch/optim/rprop.py index 86705e6ad4877..d24c407f7ba5c 100644 --- a/torch/optim/rprop.py +++ b/torch/optim/rprop.py @@ -18,6 +18,11 @@ class Rprop(Optimizer): """ def __init__(self, params, lr=1e-2, etas=(0.5, 1.2), step_sizes=(1e-6, 50)): + if not 0.0 <= lr: + raise ValueError("Invalid learning rate: {}".format(lr)) + if not 0.0 < etas[0] < 1.0 < etas[1]: + raise ValueError("Invalid eta values: {}, {}".format(etas[0], etas[1])) + defaults = dict(lr=lr, etas=etas, step_sizes=step_sizes) super(Rprop, self).__init__(params, defaults) diff --git a/torch/optim/sgd.py b/torch/optim/sgd.py index 349b6885decfb..f6d5a484efae0 100644 --- a/torch/optim/sgd.py +++ b/torch/optim/sgd.py @@ -50,6 +50,13 @@ class SGD(Optimizer): def __init__(self, params, lr=required, momentum=0, dampening=0, weight_decay=0, nesterov=False): + if not 0.0 <= lr: + raise ValueError("Invalid learning rate: {}".format(lr)) + if not 0.0 <= momentum: + raise ValueError("Invalid momentum value: {}".format(momentum)) + if not 0.0 <= weight_decay: + raise ValueError("Invalid weight_decay value: {}".format(weight_decay)) + defaults = dict(lr=lr, momentum=momentum, dampening=dampening, weight_decay=weight_decay, nesterov=nesterov) if nesterov and (momentum <= 0 or dampening != 0): diff --git a/torch/optim/sparse_adam.py b/torch/optim/sparse_adam.py index 463436d4c4082..74d15a937861e 100644 --- a/torch/optim/sparse_adam.py +++ b/torch/optim/sparse_adam.py @@ -23,6 +23,14 @@ class SparseAdam(Optimizer): """ def __init__(self, params, lr=1e-3, betas=(0.9, 0.999), eps=1e-8): + if not 0.0 < lr: + raise ValueError("Invalid learning rate: {}".format(lr)) + if not 0.0 < eps: + raise ValueError("Invalid epsilon value: {}".format(eps)) + if not 0.0 <= betas[0] < 1.0: + raise ValueError("Invalid beta parameter at index 0: {}".format(betas[0])) + if not 0.0 <= betas[1] < 1.0: + raise ValueError("Invalid beta parameter at index 1: {}".format(betas[1])) defaults = dict(lr=lr, betas=betas, eps=eps) super(SparseAdam, self).__init__(params, defaults)