It works fine on my device, I cannot figure out where is wrong.
The log is very lengthy, but it seems that some dependencies are wrong.
2025-06-15 21:52:25.777
Exception in thread Thread-2 (_routing_loop):
2025-06-15 21:52:25.777
Traceback (most recent call last):
2025-06-15 21:52:25.777
File "/usr/local/lib/python3.10/threading.py", line 1016, in _bootstrap_inner
2025-06-15 21:52:25.777
self.run()
2025-06-15 21:52:25.777
File "/usr/local/lib/python3.10/threading.py", line 953, in run
2025-06-15 21:52:25.777
self._target(*self._args, **self._kwargs)
2025-06-15 21:52:25.777
File "/usr/local/lib/python3.10/site-packages/aicrowd_gym/clients/base_oracle_client.py", line 250, in _routing_loop
2025-06-15 21:52:25.777
raise e
2025-06-15 21:52:25.777
File "/usr/local/lib/python3.10/site-packages/aicrowd_gym/clients/base_oracle_client.py", line 242, in _routing_loop
2025-06-15 21:52:25.777
raise raw_response["data"]
2025-06-15 21:52:25.777
File "/usr/local/lib/python3.10/site-packages/aicrowd_gym/clients/base_oracle_client.py", line 269, in run_agent
2025-06-15 21:52:25.777
self._agent_executor()
2025-06-15 21:52:25.777
File "/usr/local/lib/python3.10/site-packages/aicrowd_gym/clients/base_oracle_client.py", line 323, in _agent_executor
2025-06-15 21:52:25.778
result = self.execute(target_attribute, *args, **kwargs)
2025-06-15 21:52:25.778
File "/usr/local/lib/python3.10/site-packages/aicrowd_gym/clients/base_oracle_client.py", line 173, in execute
2025-06-15 21:52:25.778
return method(*args, **kwargs)
2025-06-15 21:52:25.778
File "/aicrowd-source/launcher.py", line 126, in batch_generate_response
2025-06-15 21:52:25.778
return run_with_timeout(
2025-06-15 21:52:25.778
File "/aicrowd-source/launcher.py", line 162, in run_with_timeout
2025-06-15 21:52:25.778
return fn(*args, **kwargs)
2025-06-15 21:52:25.778
File "/aicrowd-source/agents/new_agent_bert_cls.py", line 435, in batch_generate_response
2025-06-15 21:52:25.778
domains = [self.domain_cls_bert.predict(query)["label"] for query in queries]
2025-06-15 21:52:25.778
File "/aicrowd-source/agents/new_agent_bert_cls.py", line 435, in <listcomp>
2025-06-15 21:52:25.778
domains = [self.domain_cls_bert.predict(query)["label"] for query in queries]
2025-06-15 21:52:25.778
File "/aicrowd-source/agents/domain_cls_bert.py", line 29, in predict
2025-06-15 21:52:25.778
outputs = self.model(**inputs)
2025-06-15 21:52:25.778
File "/usr/local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1739, in _wrapped_call_impl
2025-06-15 21:52:25.778
return self._call_impl(*args, **kwargs)
2025-06-15 21:52:25.778
File "/usr/local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1750, in _call_impl
2025-06-15 21:52:25.779
return forward_call(*args, **kwargs)
2025-06-15 21:52:25.779
File "/usr/local/lib/python3.10/site-packages/transformers/models/modernbert/modeling_modernbert.py", line 1166, in forward
2025-06-15 21:52:25.779
outputs = self.model(
2025-06-15 21:52:25.779
File "/usr/local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1739, in _wrapped_call_impl
2025-06-15 21:52:25.779
return self._call_impl(*args, **kwargs)
2025-06-15 21:52:25.779
File "/usr/local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1750, in _call_impl
2025-06-15 21:52:25.779
return forward_call(*args, **kwargs)
2025-06-15 21:52:25.779
File "/usr/local/lib/python3.10/site-packages/transformers/models/modernbert/modeling_modernbert.py", line 863, in forward
2025-06-15 21:52:25.779
hidden_states = self.embeddings(input_ids=input_ids, inputs_embeds=inputs_embeds)
2025-06-15 21:52:25.779
File "/usr/local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1739, in _wrapped_call_impl
2025-06-15 21:52:25.780
return self._call_impl(*args, **kwargs)
2025-06-15 21:52:25.780
File "/usr/local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1750, in _call_impl
2025-06-15 21:52:25.780
return forward_call(*args, **kwargs)
2025-06-15 21:52:25.780
File "/usr/local/lib/python3.10/site-packages/transformers/models/modernbert/modeling_modernbert.py", line 215, in forward
2025-06-15 21:52:25.780
self.compiled_embeddings(input_ids)
2025-06-15 21:52:25.780
File "/usr/local/lib/python3.10/site-packages/torch/_dynamo/eval_frame.py", line 574, in _fn
2025-06-15 21:52:25.780
return fn(*args, **kwargs)
2025-06-15 21:52:25.780
File "/usr/local/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py", line 1380, in __call__
2025-06-15 21:52:25.780
return self._torchdynamo_orig_callable(
2025-06-15 21:52:25.780
File "/usr/local/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py", line 1164, in __call__
2025-06-15 21:52:25.780
result = self._inner_convert(
2025-06-15 21:52:25.780
File "/usr/local/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py", line 547, in __call__
2025-06-15 21:52:25.780
return _compile(
2025-06-15 21:52:25.780
File "/usr/local/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py", line 986, in _compile
2025-06-15 21:52:25.781
guarded_code = compile_inner(code, one_graph, hooks, transform)
2025-06-15 21:52:25.781
File "/usr/local/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py", line 715, in compile_inner
2025-06-15 21:52:25.781
return _compile_inner(code, one_graph, hooks, transform)
2025-06-15 21:52:25.781
File "/usr/local/lib/python3.10/site-packages/torch/_utils_internal.py", line 95, in wrapper_function
2025-06-15 21:52:25.781
return function(*args, **kwargs)
2025-06-15 21:52:25.781
File "/usr/local/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py", line 750, in _compile_inner
2025-06-15 21:52:25.781
out_code = transform_code_object(code, transform)
2025-06-15 21:52:25.781
File "/usr/local/lib/python3.10/site-packages/torch/_dynamo/bytecode_transformation.py", line 1361, in transform_code_object
2025-06-15 21:52:25.781
transformations(instructions, code_options)
2025-06-15 21:52:25.781
File "/usr/local/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py", line 231, in _fn
2025-06-15 21:52:25.781
return fn(*args, **kwargs)
2025-06-15 21:52:25.781
File "/usr/local/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py", line 662, in transform
2025-06-15 21:52:25.781
tracer.run()
2025-06-15 21:52:25.781
File "/usr/local/lib/python3.10/site-packages/torch/_dynamo/symbolic_convert.py", line 2868, in run
2025-06-15 21:52:25.782
super().run()
2025-06-15 21:52:25.782
File "/usr/local/lib/python3.10/site-packages/torch/_dynamo/symbolic_convert.py", line 1052, in run
2025-06-15 21:52:25.782
while self.step():
2025-06-15 21:52:25.782
File "/usr/local/lib/python3.10/site-packages/torch/_dynamo/symbolic_convert.py", line 962, in step
2025-06-15 21:52:25.782
self.dispatch_table[inst.opcode](self, inst)
2025-06-15 21:52:25.782
File "/usr/local/lib/python3.10/site-packages/torch/_dynamo/symbolic_convert.py", line 3048, in RETURN_VALUE
2025-06-15 21:52:25.782
self._return(inst)
2025-06-15 21:52:25.782
File "/usr/local/lib/python3.10/site-packages/torch/_dynamo/symbolic_convert.py", line 3033, in _return
2025-06-15 21:52:25.782
self.output.compile_subgraph(
2025-06-15 21:52:25.782
File "/usr/local/lib/python3.10/site-packages/torch/_dynamo/output_graph.py", line 1101, in compile_subgraph
2025-06-15 21:52:25.783
self.compile_and_call_fx_graph(
2025-06-15 21:52:25.783
File "/usr/local/lib/python3.10/site-packages/torch/_dynamo/output_graph.py", line 1382, in compile_and_call_fx_graph
2025-06-15 21:52:25.783
compiled_fn = self.call_user_compiler(gm)
2025-06-15 21:52:25.783
File "/usr/local/lib/python3.10/site-packages/torch/_dynamo/output_graph.py", line 1432, in call_user_compiler
2025-06-15 21:52:25.783
return self._call_user_compiler(gm)
2025-06-15 21:52:25.783
File "/usr/local/lib/python3.10/site-packages/torch/_dynamo/output_graph.py", line 1483, in _call_user_compiler
2025-06-15 21:52:25.783
raise BackendCompilerFailed(self.compiler_fn, e).with_traceback(
2025-06-15 21:52:25.783
File "/usr/local/lib/python3.10/site-packages/torch/_dynamo/output_graph.py", line 1462, in _call_user_compiler
2025-06-15 21:52:25.783
compiled_fn = compiler_fn(gm, self.example_inputs())
2025-06-15 21:52:25.783
File "/usr/local/lib/python3.10/site-packages/torch/_dynamo/repro/after_dynamo.py", line 130, in __call__
2025-06-15 21:52:25.783
compiled_gm = compiler_fn(gm, example_inputs)
2025-06-15 21:52:25.783
File "/usr/local/lib/python3.10/site-packages/torch/__init__.py", line 2340, in __call__
2025-06-15 21:52:25.784
return compile_fx(model_, inputs_, config_patches=self.config)
2025-06-15 21:52:25.784
File "/usr/local/lib/python3.10/site-packages/torch/_inductor/compile_fx.py", line 1863, in compile_fx
2025-06-15 21:52:25.784
return aot_autograd(
2025-06-15 21:52:25.784
File "/usr/local/lib/python3.10/site-packages/torch/_dynamo/backends/common.py", line 83, in __call__
2025-06-15 21:52:25.784
cg = aot_module_simplified(gm, example_inputs, **self.kwargs)
2025-06-15 21:52:25.784
File "/usr/local/lib/python3.10/site-packages/torch/_functorch/aot_autograd.py", line 1155, in aot_module_simplified
2025-06-15 21:52:25.784
compiled_fn = dispatch_and_compile()
2025-06-15 21:52:25.784
File "/usr/local/lib/python3.10/site-packages/torch/_functorch/aot_autograd.py", line 1131, in dispatch_and_compile
2025-06-15 21:52:25.784
compiled_fn, _ = create_aot_dispatcher_function(
2025-06-15 21:52:25.784
File "/usr/local/lib/python3.10/site-packages/torch/_functorch/aot_autograd.py", line 580, in create_aot_dispatcher_function
2025-06-15 21:52:25.784
return _create_aot_dispatcher_function(
2025-06-15 21:52:25.784
File "/usr/local/lib/python3.10/site-packages/torch/_functorch/aot_autograd.py", line 830, in _create_aot_dispatcher_function
2025-06-15 21:52:25.785
compiled_fn, fw_metadata = compiler_fn(
2025-06-15 21:52:25.785
File "/usr/local/lib/python3.10/site-packages/torch/_functorch/_aot_autograd/jit_compile_runtime_wrappers.py", line 203, in aot_dispatch_base
2025-06-15 21:52:25.785
compiled_fw = compiler(fw_module, updated_flat_args)
2025-06-15 21:52:25.785
File "/usr/local/lib/python3.10/site-packages/torch/_functorch/aot_autograd.py", line 489, in __call__
2025-06-15 21:52:25.785
return self.compiler_fn(gm, example_inputs)
2025-06-15 21:52:25.785
File "/usr/local/lib/python3.10/site-packages/torch/_inductor/compile_fx.py", line 1741, in fw_compiler_base
2025-06-15 21:52:25.785
return inner_compile(
2025-06-15 21:52:25.785
File "/usr/local/lib/python3.10/site-packages/torch/_inductor/compile_fx.py", line 569, in compile_fx_inner
2025-06-15 21:52:25.785
return wrap_compiler_debug(_compile_fx_inner, compiler_name="inductor")(
2025-06-15 21:52:25.785
File "/usr/local/lib/python3.10/site-packages/torch/_dynamo/repro/after_aot.py", line 102, in debug_wrapper
2025-06-15 21:52:25.785
inner_compiled_fn = compiler_fn(gm, example_inputs)
2025-06-15 21:52:25.785
File "/usr/local/lib/python3.10/site-packages/torch/_inductor/compile_fx.py", line 685, in _compile_fx_inner
2025-06-15 21:52:25.785
mb_compiled_graph = fx_codegen_and_compile(
2025-06-15 21:52:25.785
File "/usr/local/lib/python3.10/site-packages/torch/_inductor/compile_fx.py", line 1129, in fx_codegen_and_compile
2025-06-15 21:52:25.785
return scheme.codegen_and_compile(gm, example_inputs, inputs_to_check, graph_kwargs)
2025-06-15 21:52:25.785
File "/usr/local/lib/python3.10/site-packages/torch/_inductor/compile_fx.py", line 1044, in codegen_and_compile
2025-06-15 21:52:25.785
compiled_fn = graph.compile_to_module().call
2025-06-15 21:52:25.785
File "/usr/local/lib/python3.10/site-packages/torch/_inductor/graph.py", line 2027, in compile_to_module
2025-06-15 21:52:25.786
return self._compile_to_module()
2025-06-15 21:52:25.786
File "/usr/local/lib/python3.10/site-packages/torch/_inductor/graph.py", line 2033, in _compile_to_module
2025-06-15 21:52:25.786
self.codegen_with_cpp_wrapper() if self.cpp_wrapper else self.codegen()
2025-06-15 21:52:25.786
File "/usr/local/lib/python3.10/site-packages/torch/_inductor/graph.py", line 1968, in codegen
2025-06-15 21:52:25.786
self.scheduler.codegen()
2025-06-15 21:52:25.786
File "/usr/local/lib/python3.10/site-packages/torch/_inductor/scheduler.py", line 3477, in codegen
2025-06-15 21:52:25.787
return self._codegen()
2025-06-15 21:52:25.787
File "/usr/local/lib/python3.10/site-packages/torch/_inductor/scheduler.py", line 3554, in _codegen
2025-06-15 21:52:25.787
self.get_backend(device).codegen_node(node)
2025-06-15 21:52:25.787
File "/usr/local/lib/python3.10/site-packages/torch/_inductor/codegen/cuda_combined_scheduling.py", line 80, in codegen_node
2025-06-15 21:52:25.787
return self._triton_scheduling.codegen_node(node)
2025-06-15 21:52:25.787
File "/usr/local/lib/python3.10/site-packages/torch/_inductor/codegen/simd.py", line 1219, in codegen_node
2025-06-15 21:52:25.787
return self.codegen_node_schedule(
2025-06-15 21:52:25.787
File "/usr/local/lib/python3.10/site-packages/torch/_inductor/codegen/simd.py", line 1263, in codegen_node_schedule
2025-06-15 21:52:25.787
src_code = kernel.codegen_kernel()
2025-06-15 21:52:25.787
File "/usr/local/lib/python3.10/site-packages/torch/_inductor/codegen/triton.py", line 3154, in codegen_kernel
2025-06-15 21:52:25.788
**self.inductor_meta_common(),
2025-06-15 21:52:25.788
File "/usr/local/lib/python3.10/site-packages/torch/_inductor/codegen/triton.py", line 3013, in inductor_meta_common
2025-06-15 21:52:25.788
"backend_hash": torch.utils._triton.triton_hash_with_backend(),
2025-06-15 21:52:25.788
File "/usr/local/lib/python3.10/site-packages/torch/utils/_triton.py", line 111, in triton_hash_with_backend
2025-06-15 21:52:25.788
backend = triton_backend()
2025-06-15 21:52:25.788
File "/usr/local/lib/python3.10/site-packages/torch/utils/_triton.py", line 103, in triton_backend
2025-06-15 21:52:25.788
target = driver.active.get_current_target()
2025-06-15 21:52:25.788
File "/usr/local/lib/python3.10/site-packages/triton/runtime/driver.py", line 23, in __getattr__
2025-06-15 21:52:25.788
self._initialize_obj()
2025-06-15 21:52:25.788
File "/usr/local/lib/python3.10/site-packages/triton/runtime/driver.py", line 20, in _initialize_obj
2025-06-15 21:52:25.788
self._obj = self._init_fn()
2025-06-15 21:52:25.788
File "/usr/local/lib/python3.10/site-packages/triton/runtime/driver.py", line 9, in _create_driver
2025-06-15 21:52:25.788
return actives[0]()
2025-06-15 21:52:25.788
File "/usr/local/lib/python3.10/site-packages/triton/backends/nvidia/driver.py", line 450, in __init__
2025-06-15 21:52:25.788
self.utils = CudaUtils() # TODO: make static
2025-06-15 21:52:25.788
File "/usr/local/lib/python3.10/site-packages/triton/backends/nvidia/driver.py", line 80, in __init__
2025-06-15 21:52:25.788
mod = compile_module_from_src(Path(os.path.join(dirname, "driver.c")).read_text(), "cuda_utils")
2025-06-15 21:52:25.788
File "/usr/local/lib/python3.10/site-packages/triton/backends/nvidia/driver.py", line 57, in compile_module_from_src
2025-06-15 21:52:25.788
so = _build(name, src_path, tmpdir, library_dirs(), include_dir, libraries)
2025-06-15 21:52:25.788
File "/usr/local/lib/python3.10/site-packages/triton/runtime/build.py", line 32, in _build
2025-06-15 21:52:25.789
raise RuntimeError("Failed to find C compiler. Please specify via CC environment variable.")
2025-06-15 21:52:25.789
torch._dynamo.exc.BackendCompilerFailed: backend='inductor' raised:
2025-06-15 21:52:25.789
RuntimeError: Failed to find C compiler. Please specify via CC environment variable.
2025-06-15 21:52:25.789
2025-06-15 21:52:25.789
Set TORCH_LOGS="+dynamo" and TORCHDYNAMO_VERBOSE=1 for more information