runtime error

Exit code: 1. Reason: def forward(ctx, xz, conv1d_weight, conv1d_bias, x_proj_weight, delta_proj_weight, /usr/local/lib/python3.10/site-packages/mamba_ssm/ops/selective_scan_interface.py:240: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. def backward(ctx, dout): Traceback (most recent call last): File "/app/app.py", line 31, in <module> from models.generator_SEMamba_time_d4 import SEMamba File "/app/models/generator_SEMamba_time_d4.py", line 13, in <module> from .mamba_block2_SEMamba import TFMambaBlock File "/app/models/mamba_block2_SEMamba.py", line 16, in <module> from mamba_ssm import Mamba File "/usr/local/lib/python3.10/site-packages/mamba_ssm/__init__.py", line 4, in <module> from mamba_ssm.modules.mamba_simple import Mamba File "/usr/local/lib/python3.10/site-packages/mamba_ssm/modules/mamba_simple.py", line 26, in <module> from mamba_ssm.ops.triton.layer_norm import RMSNorm, layer_norm_fn, rms_norm_fn File "/usr/local/lib/python3.10/site-packages/mamba_ssm/ops/triton/layer_norm.py", line 179, in <module> def _layer_norm_fwd_1pass_kernel( File "/usr/local/lib/python3.10/site-packages/triton/runtime/autotuner.py", line 368, in decorator return Autotuner(fn, fn.arg_names, configs, key, reset_to_zero, restore_value, pre_hook=pre_hook, File "/usr/local/lib/python3.10/site-packages/triton/runtime/autotuner.py", line 130, in __init__ self.do_bench = driver.active.get_benchmarker() File "/usr/local/lib/python3.10/site-packages/triton/runtime/driver.py", line 23, in __getattr__ self._initialize_obj() File "/usr/local/lib/python3.10/site-packages/triton/runtime/driver.py", line 20, in _initialize_obj self._obj = self._init_fn() File "/usr/local/lib/python3.10/site-packages/triton/runtime/driver.py", line 8, in _create_driver raise RuntimeError(f"{len(actives)} active drivers ({actives}). There should only be one.") RuntimeError: 0 active drivers ([]). There should only be one.

Container logs: