0/? [00:00<?, ?it/s]Traceback (most recent call last):
File "/workspace/assethub-ml-server/api/common/libs/USD/launch.py", line 240, in <module>
main(args, extras)
File "/workspace/assethub-ml-server/api/common/libs/USD/launch.py", line 183, in main
trainer.fit(system, datamodule=dm, ckpt_path=cfg.resume)
File "/workspace/assethub-ml-server/api/common/libs/ImageDream/venv_image/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 543, in fit
call._call_and_handle_interrupt(
File "/workspace/assethub-ml-server/api/common/libs/ImageDream/venv_image/lib/python3.10/site-packages/pytorch_lightning/trainer/call.py", line 44, in _call_and_handle_interrupt
return trainer_fn(*args, **kwargs)
File "/workspace/assethub-ml-server/api/common/libs/ImageDream/venv_image/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 579, in _fit_impl
self._run(model, ckpt_path=ckpt_path)
File "/workspace/assethub-ml-server/api/common/libs/ImageDream/venv_image/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 986, in _run
results = self._run_stage()
File "/workspace/assethub-ml-server/api/common/libs/ImageDream/venv_image/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 1032, in _run_stage
self.fit_loop.run()
File "/workspace/assethub-ml-server/api/common/libs/ImageDream/venv_image/lib/python3.10/site-packages/pytorch_lightning/loops/fit_loop.py", line 205, in run
self.advance()
File "/workspace/assethub-ml-server/api/common/libs/ImageDream/venv_image/lib/python3.10/site-packages/pytorch_lightning/loops/fit_loop.py", line 363, in advance
self.epoch_loop.run(self._data_fetcher)
File "/workspace/assethub-ml-server/api/common/libs/ImageDream/venv_image/lib/python3.10/site-packages/pytorch_lightning/loops/training_epoch_loop.py", line 138, in run
self.advance(data_fetcher)
File "/workspace/assethub-ml-server/api/common/libs/ImageDream/venv_image/lib/python3.10/site-packages/pytorch_lightning/loops/training_epoch_loop.py", line 242, in advance
batch_output = self.automatic_optimization.run(trainer.optimizers[0], batch_idx, kwargs)
File "/workspace/assethub-ml-server/api/common/libs/ImageDream/venv_image/lib/python3.10/site-packages/pytorch_lightning/loops/optimization/automatic.py", line 191, in run
self._optimizer_step(batch_idx, closure)
File "/workspace/assethub-ml-server/api/common/libs/ImageDream/venv_image/lib/python3.10/site-packages/pytorch_lightning/loops/optimization/automatic.py", line 269, in _optimizer_step
call._call_lightning_module_hook(
File "/workspace/assethub-ml-server/api/common/libs/ImageDream/venv_image/lib/python3.10/site-packages/pytorch_lightning/trainer/call.py", line 157, in _call_lightning_module_hook
output = fn(*args, **kwargs)
File "/workspace/assethub-ml-server/api/common/libs/ImageDream/venv_image/lib/python3.10/site-packages/pytorch_lightning/core/module.py", line 1303, in optimizer_step
optimizer.step(closure=optimizer_closure)
File "/workspace/assethub-ml-server/api/common/libs/ImageDream/venv_image/lib/python3.10/site-packages/pytorch_lightning/core/optimizer.py", line 152, in step
step_output = self._strategy.optimizer_step(self._optimizer, closure, **kwargs)
File "/workspace/assethub-ml-server/api/common/libs/ImageDream/venv_image/lib/python3.10/site-packages/pytorch_lightning/strategies/strategy.py", line 239, in optimizer_step
return self.precision_plugin.optimizer_step(optimizer, model=model, closure=closure, **kwargs)
File "/workspace/assethub-ml-server/api/common/libs/ImageDream/venv_image/lib/python3.10/site-packages/pytorch_lightning/plugins/precision/precision.py", line 122, in optimizer_step
return optimizer.step(closure=closure, **kwargs)
File "/workspace/assethub-ml-server/api/common/libs/ImageDream/venv_image/lib/python3.10/site-packages/torch/optim/optimizer.py", line 373, in wrapper
out = func(*args, **kwargs)
File "/workspace/assethub-ml-server/api/common/libs/ImageDream/venv_image/lib/python3.10/site-packages/torch/optim/optimizer.py", line 76, in _use_grad
ret = func(self, *args, **kwargs)
File "/workspace/assethub-ml-server/api/common/libs/ImageDream/venv_image/lib/python3.10/site-packages/torch/optim/adamw.py", line 161, in step
loss = closure()
File "/workspace/assethub-ml-server/api/common/libs/ImageDream/venv_image/lib/python3.10/site-packages/pytorch_lightning/plugins/precision/precision.py", line 108, in _wrap_closure
closure_result = closure()
File "/workspace/assethub-ml-server/api/common/libs/ImageDream/venv_image/lib/python3.10/site-packages/pytorch_lightning/loops/optimization/automatic.py", line 144, in __call__
self._result = self.closure(*args, **kwargs)
File "/workspace/assethub-ml-server/api/common/libs/ImageDream/venv_image/lib/python3.10/site-packages/torch/utils/_contextlib.py", line 115, in decorate_context
return func(*args, **kwargs)
File "/workspace/assethub-ml-server/api/common/libs/ImageDream/venv_image/lib/python3.10/site-packages/pytorch_lightning/loops/optimization/automatic.py", line 129, in closure
step_output = self._step_fn()
File "/workspace/assethub-ml-server/api/common/libs/ImageDream/venv_image/lib/python3.10/site-packages/pytorch_lightning/loops/optimization/automatic.py", line 319, in _training_step
training_step_output = call._call_strategy_hook(trainer, "training_step", *kwargs.values())
File "/workspace/assethub-ml-server/api/common/libs/ImageDream/venv_image/lib/python3.10/site-packages/pytorch_lightning/trainer/call.py", line 309, in _call_strategy_hook
output = fn(*args, **kwargs)
File "/workspace/assethub-ml-server/api/common/libs/ImageDream/venv_image/lib/python3.10/site-packages/pytorch_lightning/strategies/strategy.py", line 391, in training_step
return self.lightning_module.training_step(*args, **kwargs)
File "/workspace/assethub-ml-server/api/common/libs/USD/threestudio/systems/usd.py", line 68, in training_step
guidance_out = self.guidance(
File "/workspace/assethub-ml-server/api/common/libs/ImageDream/venv_image/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1518, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
File "/workspace/assethub-ml-server/api/common/libs/ImageDream/venv_image/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1527, in _call_impl
return forward_call(*args, **kwargs)
File "/workspace/assethub-ml-server/api/common/libs/USD/threestudio/models/guidance/usd_guidance.py", line 411, in forward
grad = self.compute_grad_usd(
File "/workspace/assethub-ml-server/api/common/libs/USD/threestudio/models/guidance/usd_guidance.py", line 304, in compute_grad_usd
noise_pred = self.forward_unet(
File "/workspace/assethub-ml-server/api/common/libs/ImageDream/venv_image/lib/python3.10/site-packages/torch/amp/autocast_mode.py", line 16, in decorate_autocast
return func(*args, **kwargs)
File "/workspace/assethub-ml-server/api/common/libs/USD/threestudio/models/guidance/usd_guidance.py", line 189, in forward_unet
return unet(
File "/workspace/assethub-ml-server/api/common/libs/ImageDream/venv_image/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1518, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
File "/workspace/assethub-ml-server/api/common/libs/ImageDream/venv_image/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1527, in _call_impl
return forward_call(*args, **kwargs)
File "/workspace/assethub-ml-server/api/common/libs/ImageDream/venv_image/lib/python3.10/site-packages/diffusers/models/unet_2d_condition.py", line 905, in forward
sample, res_samples = downsample_block(
File "/workspace/assethub-ml-server/api/common/libs/ImageDream/venv_image/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1518, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
File "/workspace/assethub-ml-server/api/common/libs/ImageDream/venv_image/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1527, in _call_impl
return forward_call(*args, **kwargs)
File "/workspace/assethub-ml-server/api/common/libs/ImageDream/venv_image/lib/python3.10/site-packages/diffusers/models/unet_2d_blocks.py", line 993, in forward
hidden_states = attn(
File "/workspace/assethub-ml-server/api/common/libs/ImageDream/venv_image/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1518, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
File "/workspace/assethub-ml-server/api/common/libs/ImageDream/venv_image/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1527, in _call_impl
return forward_call(*args, **kwargs)
File "/workspace/assethub-ml-server/api/common/libs/ImageDream/venv_image/lib/python3.10/site-packages/diffusers/models/transformer_2d.py", line 291, in forward
hidden_states = block(
File "/workspace/assethub-ml-server/api/common/libs/ImageDream/venv_image/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1518, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
File "/workspace/assethub-ml-server/api/common/libs/ImageDream/venv_image/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1527, in _call_impl
return forward_call(*args, **kwargs)
File "/workspace/assethub-ml-server/api/common/libs/ImageDream/venv_image/lib/python3.10/site-packages/diffusers/models/attention.py", line 170, in forward
attn_output = self.attn2(
File "/workspace/assethub-ml-server/api/common/libs/ImageDream/venv_image/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1518, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
File "/workspace/assethub-ml-server/api/common/libs/ImageDream/venv_image/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1527, in _call_impl
return forward_call(*args, **kwargs)
File "/workspace/assethub-ml-server/api/common/libs/ImageDream/venv_image/lib/python3.10/site-packages/diffusers/models/attention_processor.py", line 321, in forward
return self.processor(
File "/workspace/assethub-ml-server/api/common/libs/ImageDream/venv_image/lib/python3.10/site-packages/diffusers/models/attention_processor.py", line 1117, in __call__
key = attn.to_k(encoder_hidden_states)
File "/workspace/assethub-ml-server/api/common/libs/ImageDream/venv_image/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1518, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
File "/workspace/assethub-ml-server/api/common/libs/ImageDream/venv_image/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1527, in _call_impl
return forward_call(*args, **kwargs)
File "/workspace/assethub-ml-server/api/common/libs/ImageDream/venv_image/lib/python3.10/site-packages/torch/nn/modules/linear.py", line 114, in forward
return F.linear(input, self.weight, self.bias)
RuntimeError: mat1 and mat2 shapes cannot be multiplied (154x768 and 1024x320)
Hi, thank you for your excellent work!
I tried it, but I was faced with this error every time.
How can we solve it?
Thanks
RuntimeError: mat1 and mat2 shapes cannot be multiplied (154x768 and 1024x320)