Skip to content

Can a single gpu be used for training? #18

Description

@AIzhiqing

When I execute python iso/scripts/train_iso.py n_gpus=1 batch_size=2, the error is as follows:

 trainer.fit(model, data_module) 
 File "/opt/conda/envs/iso/lib/python3.9/site-packages/pytorch_lightning/trainer/trainer.py", line 520, in fit
   call._call_and_handle_interrupt(
 File "/opt/conda/envs/iso/lib/python3.9/site-packages/pytorch_lightning/trainer/call.py", line 42, in _call_and_handle_interrupt
   return trainer.strategy.launcher.launch(trainer_fn, *args, trainer=trainer, **kwargs)
 File "/opt/conda/envs/iso/lib/python3.9/site-packages/pytorch_lightning/strategies/launchers/subprocess_script.py", line 92, in launch
   return function(*args, **kwargs)
 File "/opt/conda/envs/iso/lib/python3.9/site-packages/pytorch_lightning/trainer/trainer.py", line 559, in _fit_impl
   self._run(model, ckpt_path=ckpt_path)
 File "/opt/conda/envs/iso/lib/python3.9/site-packages/pytorch_lightning/trainer/trainer.py", line 935, in _run
   results = self._run_stage()
 File "/opt/conda/envs/iso/lib/python3.9/site-packages/pytorch_lightning/trainer/trainer.py", line 978, in _run_stage
   self.fit_loop.run()
 File "/opt/conda/envs/iso/lib/python3.9/site-packages/pytorch_lightning/loops/fit_loop.py", line 201, in run
   self.advance()
 File "/opt/conda/envs/iso/lib/python3.9/site-packages/pytorch_lightning/loops/fit_loop.py", line 354, in advance
   self.epoch_loop.run(self._data_fetcher)
 File "/opt/conda/envs/iso/lib/python3.9/site-packages/pytorch_lightning/loops/training_epoch_loop.py", line 133, in run
   self.advance(data_fetcher)
 File "/opt/conda/envs/iso/lib/python3.9/site-packages/pytorch_lightning/loops/training_epoch_loop.py", line 218, in advance
   batch_output = self.automatic_optimization.run(trainer.optimizers[0], kwargs)
 File "/opt/conda/envs/iso/lib/python3.9/site-packages/pytorch_lightning/loops/optimization/automatic.py", line 185, in run
   self._optimizer_step(kwargs.get("batch_idx", 0), closure)
 File "/opt/conda/envs/iso/lib/python3.9/site-packages/pytorch_lightning/loops/optimization/automatic.py", line 261, in _optimizer_step
   call._call_lightning_module_hook(
 File "/opt/conda/envs/iso/lib/python3.9/site-packages/pytorch_lightning/trainer/call.py", line 142, in _call_lightning_module_hook
   output = fn(*args, **kwargs)
 File "/opt/conda/envs/iso/lib/python3.9/site-packages/pytorch_lightning/core/module.py", line 1266, in optimizer_step
   optimizer.step(closure=optimizer_closure)
 File "/opt/conda/envs/iso/lib/python3.9/site-packages/pytorch_lightning/core/optimizer.py", line 158, in step
   step_output = self._strategy.optimizer_step(self._optimizer, closure, **kwargs)
 File "/opt/conda/envs/iso/lib/python3.9/site-packages/pytorch_lightning/strategies/ddp.py", line 257, in optimizer_step
   optimizer_output = super().optimizer_step(optimizer, closure, model, **kwargs)
 File "/opt/conda/envs/iso/lib/python3.9/site-packages/pytorch_lightning/strategies/strategy.py", line 224, in optimizer_step
   return self.precision_plugin.optimizer_step(optimizer, model=model, closure=closure, **kwargs)
 File "/opt/conda/envs/iso/lib/python3.9/site-packages/pytorch_lightning/plugins/precision/precision_plugin.py", line 114, in optimizer_step
   return optimizer.step(closure=closure, **kwargs)
 File "/opt/conda/envs/iso/lib/python3.9/site-packages/torch/optim/lr_scheduler.py", line 69, in wrapper
   return wrapped(*args, **kwargs)
 File "/opt/conda/envs/iso/lib/python3.9/site-packages/torch/optim/optimizer.py", line 280, in wrapper
   out = func(*args, **kwargs)
 File "/opt/conda/envs/iso/lib/python3.9/site-packages/torch/optim/optimizer.py", line 33, in _use_grad
   ret = func(self, *args, **kwargs)
 File "/opt/conda/envs/iso/lib/python3.9/site-packages/torch/optim/adamw.py", line 148, in step
   loss = closure()
 File "/opt/conda/envs/iso/lib/python3.9/site-packages/pytorch_lightning/plugins/precision/precision_plugin.py", line 101, in _wrap_closure
   closure_result = closure()
 File "/opt/conda/envs/iso/lib/python3.9/site-packages/pytorch_lightning/loops/optimization/automatic.py", line 140, in __call__
   self._result = self.closure(*args, **kwargs)
 File "/opt/conda/envs/iso/lib/python3.9/site-packages/pytorch_lightning/loops/optimization/automatic.py", line 135, in closure
   self._backward_fn(step_output.closure_loss)
 File "/opt/conda/envs/iso/lib/python3.9/site-packages/pytorch_lightning/loops/optimization/automatic.py", line 233, in backward_fn
   call._call_strategy_hook(self.trainer, "backward", loss, optimizer)
 File "/opt/conda/envs/iso/lib/python3.9/site-packages/pytorch_lightning/trainer/call.py", line 288, in _call_strategy_hook
   output = fn(*args, **kwargs)
 File "/opt/conda/envs/iso/lib/python3.9/site-packages/pytorch_lightning/strategies/strategy.py", line 199, in backward
   self.precision_plugin.backward(closure_loss, self.lightning_module, optimizer, *args, **kwargs)
 File "/opt/conda/envs/iso/lib/python3.9/site-packages/pytorch_lightning/plugins/precision/precision_plugin.py", line 67, in backward
   model.backward(tensor, *args, **kwargs)
 File "/opt/conda/envs/iso/lib/python3.9/site-packages/pytorch_lightning/core/module.py", line 1055, in backward
   loss.backward(*args, **kwargs)
 File "/opt/conda/envs/iso/lib/python3.9/site-packages/torch/_tensor.py", line 487, in backward
   torch.autograd.backward(
 File "/opt/conda/envs/iso/lib/python3.9/site-packages/torch/autograd/__init__.py", line 200, in backward
   Variable._execution_engine.run_backward(  # Calls into the C++ engine to run the backward pass
RuntimeError: GET was unable to find an engine to execute this computation

Activity

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment

Metadata

Metadata

Assignees

No one assigned

    Labels

    No labels
    No labels

    Projects

    No projects

      Milestone

      No milestone

      Relationships

      None yet

      Development

      No branches or pull requests

      Issue actions