root@gpu-prod:/home/neo/ranjith/rnd# docker compose -f gemma-4.yaml up
[+] up 1/1
Container gemma-test Recreated 0.1s
Attaching to gemma-test
gemma-test | (APIServer pid=1) INFO 10-30 12:01:03 [api_utils.py:347]
gemma-test | (APIServer pid=1) INFO 10-30 12:01:03 [api_utils.py:347] █ █ █▄ ▄█
gemma-test | (APIServer pid=1) INFO 10-30 12:01:03 [api_utils.py:347] ▄▄ ▄█ █ █ █ ▀▄▀ █ version 0.32.0
gemma-test | (APIServer pid=1) INFO 10-30 12:01:03 [api_utils.py:347] █▄█▀ █ █ █ █ model /root/.cache/huggingface/hub/models–google–gemma-4-31B/snapshots/5bbc2fb1c1b2c611d06e3d9f23c170ba21659d89
gemma-test | (APIServer pid=1) INFO 10-30 12:01:03 [api_utils.py:347] ▀▀ ▀▀▀▀▀ ▀▀▀▀▀ ▀ ▀
gemma-test | (APIServer pid=1) INFO 10-30 12:01:03 [api_utils.py:347]
gemma-test | (APIServer pid=1) INFO 10-30 12:01:03 [api_utils.py:286] non-default args: {‘model_tag’: ‘/root/.cache/huggingface/hub/models–google–gemma-4-31B/snapshots/5bbc2fb1c1b2c611d06e3d9f23c170ba21659d89’, ‘model’: ‘/root/.cache/huggingface/hub/models–google–gemma-4-31B/snapshots/5bbc2fb1c1b2c611d06e3d9f23c170ba21659d89’, ‘max_model_len’: 16304, ‘served_model_name’: [‘gemma-test’], ‘gpu_memory_utilization’: 0.8, ‘mm_processor_cache_gb’: 0.0, ‘max_num_batched_tokens’: 4096}
gemma-test | (APIServer pid=1) INFO 10-30 12:01:14 [model.py:733] Resolved architecture: Gemma4ForConditionalGeneration
gemma-test | (APIServer pid=1) INFO 10-30 12:01:14 [model.py:2152] Using max model len 16304
gemma-test | (APIServer pid=1) INFO 10-30 12:01:17 [scheduler.py:307] Chunked prefill is enabled with max_num_batched_tokens=4096.
gemma-test | (APIServer pid=1) INFO 10-30 12:01:17 [config.py:286] Gemma4 model has heterogeneous head dimensions {‘sliding_attention’: 256, ‘full_attention’: 512}. FA4 not available, forcing TRITON_ATTN backend.
gemma-test | (APIServer pid=1) INFO 10-30 12:01:17 [kernel.py:467] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=[‘native’], fused_add_rms_norm=[‘native’], gelu_and_mul_sparse=[‘triton’, ‘native’])
gemma-test | (APIServer pid=1) WARNING 10-30 12:01:17 [cuda.py:339] Forcing --disable_chunked_mm_input for models with multimodal-bidirectional attention.
gemma-test | (EngineCore pid=177) INFO 10-30 12:01:30 [core.py:133] Initializing a V1 LLM engine (v0.32.0) with config: model=‘/root/.cache/huggingface/hub/models–google–gemma-4-31B/snapshots/5bbc2fb1c1b2c611d06e3d9f23c170ba21659d89’, speculative_config=None, tokenizer=‘/root/.cache/huggingface/hub/models–google–gemma-4-31B/snapshots/5bbc2fb1c1b2c611d06e3d9f23c170ba21659d89’, skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=False, dtype=torch.bfloat16, max_seq_len=16304, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, decode_context_parallel_size=1, dcp_comm_backend=ag_rs, disable_custom_all_reduce=False, quantization=None, quantization_config=None, enforce_eager=False, aux_output_config=AuxOutputConfig(enable_return_routed_experts=False, max_bytes=None), kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend=‘auto’, disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser=‘’, reasoning_parser_plugin=‘’, enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, per_request_spec_decode_metrics=‘none’, kv_cache_metrics=False, kv_cache_metrics_sample=0.01, custom_histogram_buckets=None, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False, jit_monitor_mode=‘warn’, jit_monitor_verbose=False), seed=0, served_model_name=gemma-test, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={‘mode’: <CompilationMode.VLLM_COMPILE: 3>, ‘debug_dump_path’: None, ‘cache_dir’: ‘’, ‘compile_cache_save_format’: ‘binary’, ‘backend’: ‘inductor’, ‘custom_ops’: [‘none’], ‘ir_enable_torch_wrap’: True, ‘splitting_ops’: [‘vllm::unified_attention_with_output’, ‘vllm::unified_mla_attention_with_output’, ‘vllm::mamba_mixer2’, ‘vllm::mamba_mixer’, ‘vllm::short_conv’, ‘vllm::qwen4_exp_ple_short_conv’, ‘vllm::qwen4_exp_qsa_with_output’, ‘vllm::qwen4_exp_qsa_prepare_with_output’, ‘vllm::linear_attention’, ‘vllm::qwen_gdn_attention_core’, ‘vllm::qwen_gdn_attention_core_fused_norm_packed’, ‘vllm::gdn_attention_core_xpu’, ‘vllm::olmo_hybrid_gdn_full_forward’, ‘vllm::sparse_attn_indexer’, ‘vllm::rocm_aiter_sparse_attn_indexer’, ‘vllm::deepseek_v4_attention’, ‘vllm::hpc_rope_norm_forward’, ‘vllm::unified_kv_cache_update’, ‘vllm::unified_mla_kv_cache_update’], ‘compile_mm_encoder’: False, ‘cudagraph_mm_encoder’: False, ‘encoder_cudagraph_token_budgets’: , ‘encoder_cudagraph_max_vision_items_per_batch’: 0, ‘encoder_cudagraph_max_frames_per_batch’: None, ‘cudagraph_decoder_replay’: True, ‘decoder_replay_cudagraph_capture_sizes’: , ‘decoder_replay_trim_threshold’: 768, ‘compile_sizes’: , ‘compile_ranges_endpoints’: [4096], ‘inductor_compile_config’: {‘enable_auto_functionalized_v2’: False, ‘combo_kernels’: True, ‘benchmark_combo_kernel’: True}, ‘inductor_passes’: {}, ‘cudagraph_mode’: <CUDAGraphMode.FULL_AND_PIECEWISE: (2, 1)>, ‘cudagraph_num_of_warmups’: 1, ‘cudagraph_capture_sizes’: [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128, 136, 144, 152, 160, 168, 176, 184, 192, 200, 208, 216, 224, 232, 240, 248, 256, 272, 288, 304, 320, 336, 352, 368, 384, 400, 416, 432, 448, 464, 480, 496, 512], ‘cudagraph_copy_inputs’: False, ‘cudagraph_specialize_lora’: True, ‘use_inductor_graph_partition’: False, ‘pass_config’: {‘fuse_norm_quant’: False, ‘fuse_act_quant’: False, ‘fuse_attn_quant’: False, ‘fuse_allreduce_rms’: False, ‘enable_qk_norm_rope_fusion’: False, ‘fuse_rope_kvcache_cat_mla’: False, ‘fuse_act_padding’: False, ‘fuse_qk_norm_rope_kvcache’: False}, ‘max_cudagraph_capture_size’: 512, ‘dynamic_shapes_config’: {‘type’: <DynamicShapesType.BACKED: ‘backed’>, ‘evaluate_guards’: False, ‘assume_32_bit_indexing’: False}, ‘local_cache_dir’: None, ‘fast_moe_cold_start’: False, ‘static_all_moe_layers’: }, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=[‘native’], fused_add_rms_norm=[‘native’], gelu_and_mul_sparse=[‘triton’, ‘native’]), enable_flashinfer_autotune=True, enable_rocm_segmented_attn_autotune=False, enable_cutedsl_warmup=True, enable_jit_warmup=True, moe_backend=‘auto’, sparse_indexer_topk_backend=‘auto’, linear_backend=‘auto’, linear_backend_per_quant=None)
gemma-test | (EngineCore pid=177) INFO 10-30 12:01:33 [parallel_state.py:1906] world_size=1 rank=0 local_rank=0 distributed_init_method=file:///tmp/vllm_dist_d123bb0d81ca4dc39d5f3b0cdafd4792 backend=nccl
gemma-test | (EngineCore pid=177) INFO 10-30 12:01:33 [parallel_state.py:2350] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, ETP rank 0, EP rank N/A, EPLB rank N/A
gemma-test | (EngineCore pid=177) INFO 10-30 12:01:33 [gpu_worker.py:518] Using V2 Model Runner
gemma-test | (EngineCore pid=177) INFO 10-30 12:01:34 [model_runner.py:392] Loading model from scratch…
gemma-test | (EngineCore pid=177) INFO 10-30 12:01:34 [cuda.py:446] Using AttentionBackendEnum.TRITON_ATTN backend.
gemma-test | (EngineCore pid=177) INFO 10-30 12:01:34 [cuda.py:446] Using AttentionBackendEnum.TRITON_ATTN backend.
gemma-test | (EngineCore pid=177) INFO 10-30 12:01:36 [base_loader.py:82] Initializing model took 2.284558 seconds
gemma-test | (EngineCore pid=177) INFO 10-30 12:01:36 [weight_utils.py:1108] Filesystem type for checkpoints: EXT4. Checkpoint size: 58.25 GiB. Available RAM: 11.54 GiB.
gemma-test | (EngineCore pid=177) INFO 10-30 12:01:36 [weight_utils.py:1138] Auto-prefetch is disabled because the filesystem (EXT4) is not a recognized network FS (NFS/Lustre) and the checkpoint size (58.25 GiB) exceeds 90% of available RAM (11.54 GiB).
Loading safetensors checkpoint shards: 0% Completed | 0/2 [00:00<?, ?it/s]
Loading safetensors checkpoint shards: 0% Completed | 0/2 [00:00<?, ?it/s]
gemma-test |
gemma-test | Process EngineCore:
gemma-test | (EngineCore pid=177) ERROR 10-30 12:01:37 [core.py:1510] EngineCore failed to start.
gemma-test | (EngineCore pid=177) ERROR 10-30 12:01:37 [core.py:1510] Traceback (most recent call last):
gemma-test | (EngineCore pid=177) ERROR 10-30 12:01:37 [core.py:1510] File “/usr/local/lib/python3.12/dist-packages/vllm/v1/engine/core.py”, line 1482, in run_engine_core
gemma-test | (EngineCore pid=177) ERROR 10-30 12:01:37 [core.py:1510] engine_core = EngineCoreProc(*args, engine_index=dp_rank, **kwargs)
gemma-test | (EngineCore pid=177) ERROR 10-30 12:01:37 [core.py:1510] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
gemma-test | (EngineCore pid=177) ERROR 10-30 12:01:37 [core.py:1510] File “/usr/local/lib/python3.12/dist-packages/vllm/tracing/otel.py”, line 175, in sync_wrapper
gemma-test | (EngineCore pid=177) ERROR 10-30 12:01:37 [core.py:1510] return func(*args, **kwargs)
gemma-test | (EngineCore pid=177) ERROR 10-30 12:01:37 [core.py:1510] ^^^^^^^^^^^^^^^^^^^^^
gemma-test | (EngineCore pid=177) ERROR 10-30 12:01:37 [core.py:1510] File “/usr/local/lib/python3.12/dist-packages/vllm/v1/engine/core.py”, line 1194, in init
gemma-test | (EngineCore pid=177) ERROR 10-30 12:01:37 [core.py:1510] super().init(
gemma-test | (EngineCore pid=177) ERROR 10-30 12:01:37 [core.py:1510] File “/usr/local/lib/python3.12/dist-packages/vllm/v1/engine/core.py”, line 144, in init
gemma-test | (EngineCore pid=177) ERROR 10-30 12:01:37 [core.py:1510] self.model_executor = executor_class(vllm_config)
gemma-test | (EngineCore pid=177) ERROR 10-30 12:01:37 [core.py:1510] ^^^^^^^^^^^^^^^^^^^^^^^^^^^
gemma-test | (EngineCore pid=177) ERROR 10-30 12:01:37 [core.py:1510] File “/usr/local/lib/python3.12/dist-packages/vllm/tracing/otel.py”, line 175, in sync_wrapper
gemma-test | (EngineCore pid=177) ERROR 10-30 12:01:37 [core.py:1510] return func(*args, **kwargs)
gemma-test | (EngineCore pid=177) ERROR 10-30 12:01:37 [core.py:1510] ^^^^^^^^^^^^^^^^^^^^^
gemma-test | (EngineCore pid=177) ERROR 10-30 12:01:37 [core.py:1510] File “/usr/local/lib/python3.12/dist-packages/vllm/v1/executor/abstract.py”, line 126, in init
gemma-test | (EngineCore pid=177) ERROR 10-30 12:01:37 [core.py:1510] self._init_executor()
gemma-test | (EngineCore pid=177) ERROR 10-30 12:01:37 [core.py:1510] File “/usr/local/lib/python3.12/dist-packages/vllm/v1/executor/uniproc_executor.py”, line 74, in _init_executor
gemma-test | (EngineCore pid=177) ERROR 10-30 12:01:37 [core.py:1510] self.driver_worker.load_model()
gemma-test | (EngineCore pid=177) ERROR 10-30 12:01:37 [core.py:1510] File “/usr/local/lib/python3.12/dist-packages/vllm/v1/worker/gpu_worker.py”, line 594, in load_model
gemma-test | (EngineCore pid=177) ERROR 10-30 12:01:37 [core.py:1510] self.model_runner.load_model(load_dummy_weights=load_dummy_weights)
gemma-test | (EngineCore pid=177) ERROR 10-30 12:01:37 [core.py:1510] File “/usr/local/lib/python3.12/dist-packages/vllm/v1/worker/gpu/model_runner.py”, line 396, in load_model
gemma-test | (EngineCore pid=177) ERROR 10-30 12:01:37 [core.py:1510] self.model = model or model_loader.load_model(
gemma-test | (EngineCore pid=177) ERROR 10-30 12:01:37 [core.py:1510] ^^^^^^^^^^^^^^^^^^^^^^^^
gemma-test | (EngineCore pid=177) ERROR 10-30 12:01:37 [core.py:1510] File “/usr/local/lib/python3.12/dist-packages/vllm/tracing/otel.py”, line 175, in sync_wrapper
gemma-test | (EngineCore pid=177) ERROR 10-30 12:01:37 [core.py:1510] return func(*args, **kwargs)
gemma-test | (EngineCore pid=177) ERROR 10-30 12:01:37 [core.py:1510] ^^^^^^^^^^^^^^^^^^^^^
gemma-test | (EngineCore pid=177) ERROR 10-30 12:01:37 [core.py:1510] File “/usr/local/lib/python3.12/dist-packages/vllm/model_executor/model_loader/base_loader.py”, line 88, in load_model
gemma-test | (EngineCore pid=177) ERROR 10-30 12:01:37 [core.py:1510] self.load_weights(model, model_config)
gemma-test | (EngineCore pid=177) ERROR 10-30 12:01:37 [core.py:1510] File “/usr/local/lib/python3.12/dist-packages/vllm/tracing/otel.py”, line 175, in sync_wrapper
gemma-test | (EngineCore pid=177) ERROR 10-30 12:01:37 [core.py:1510] return func(*args, **kwargs)
gemma-test | (EngineCore pid=177) ERROR 10-30 12:01:37 [core.py:1510] ^^^^^^^^^^^^^^^^^^^^^
gemma-test | (EngineCore pid=177) ERROR 10-30 12:01:37 [core.py:1510] File “/usr/local/lib/python3.12/dist-packages/vllm/model_executor/model_loader/default_loader.py”, line 498, in load_weights
gemma-test | (EngineCore pid=177) ERROR 10-30 12:01:37 [core.py:1510] loaded_weights = model.load_weights(self.get_all_weights(model_config, model))
gemma-test | (EngineCore pid=177) ERROR 10-30 12:01:37 [core.py:1510] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
gemma-test | (EngineCore pid=177) ERROR 10-30 12:01:37 [core.py:1510] File “/usr/local/lib/python3.12/dist-packages/vllm/model_executor/models/gemma4_mm.py”, line 2228, in load_weights
gemma-test | (EngineCore pid=177) ERROR 10-30 12:01:37 [core.py:1510] return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
gemma-test | (EngineCore pid=177) ERROR 10-30 12:01:37 [core.py:1510] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
gemma-test | (EngineCore pid=177) ERROR 10-30 12:01:37 [core.py:1510] File “/usr/local/lib/python3.12/dist-packages/vllm/model_executor/model_loader/reload/torchao_decorator.py”, line 52, in patched_model_load_weights
gemma-test | (EngineCore pid=177) ERROR 10-30 12:01:37 [core.py:1510] return original_load_weights(self, *args, **kwargs)
gemma-test | (EngineCore pid=177) ERROR 10-30 12:01:37 [core.py:1510] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
gemma-test | (EngineCore pid=177) ERROR 10-30 12:01:37 [core.py:1510] File “/usr/local/lib/python3.12/dist-packages/vllm/model_executor/models/utils.py”, line 476, in load_weights
gemma-test | (EngineCore pid=177) ERROR 10-30 12:01:37 [core.py:1510] autoloaded_weights = set(self._load_module(“”, self.module, weights))
gemma-test | (EngineCore pid=177) ERROR 10-30 12:01:37 [core.py:1510] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
gemma-test | (EngineCore pid=177) ERROR 10-30 12:01:37 [core.py:1510] File “/usr/local/lib/python3.12/dist-packages/vllm/model_executor/models/utils.py”, line 409, in _load_module
gemma-test | (EngineCore pid=177) ERROR 10-30 12:01:37 [core.py:1510] for child_prefix, child_weights in self._groupby_prefix(weights):
gemma-test | (EngineCore pid=177) ERROR 10-30 12:01:37 [core.py:1510] File “/usr/local/lib/python3.12/dist-packages/vllm/model_executor/models/utils.py”, line 272, in _groupby_prefix
gemma-test | (EngineCore pid=177) ERROR 10-30 12:01:37 [core.py:1510] for prefix, group in itertools.groupby(weights_by_parts, key=lambda x: x[0][0]):
gemma-test | (EngineCore pid=177) ERROR 10-30 12:01:37 [core.py:1510] File “/usr/local/lib/python3.12/dist-packages/vllm/model_executor/models/utils.py”, line 267, in
gemma-test | (EngineCore pid=177) ERROR 10-30 12:01:37 [core.py:1510] weights_by_parts = (
gemma-test | (EngineCore pid=177) ERROR 10-30 12:01:37 [core.py:1510] ^
gemma-test | (EngineCore pid=177) ERROR 10-30 12:01:37 [core.py:1510] File “/usr/local/lib/python3.12/dist-packages/vllm/model_executor/models/utils.py”, line 483, in _filter_skipped
gemma-test | (EngineCore pid=177) ERROR 10-30 12:01:37 [core.py:1510] for name, weight in weights:
gemma-test | (EngineCore pid=177) ERROR 10-30 12:01:37 [core.py:1510] File “/usr/local/lib/python3.12/dist-packages/vllm/model_executor/models/utils.py”, line 150, in apply
gemma-test | (EngineCore pid=177) ERROR 10-30 12:01:37 [core.py:1510] for name, data in weights:
gemma-test | (EngineCore pid=177) ERROR 10-30 12:01:37 [core.py:1510] File “/usr/local/lib/python3.12/dist-packages/vllm/model_executor/model_loader/default_loader.py”, line 380, in get_all_weights
gemma-test | (EngineCore pid=177) ERROR 10-30 12:01:37 [core.py:1510] yield from self._get_weights_iterator(primary_weights)
gemma-test | (EngineCore pid=177) ERROR 10-30 12:01:37 [core.py:1510] File “/usr/local/lib/python3.12/dist-packages/vllm/model_executor/model_loader/default_loader.py”, line 365, in
gemma-test | (EngineCore pid=177) ERROR 10-30 12:01:37 [core.py:1510] return ((source.prefix + name, tensor) for (name, tensor) in weights_iterator)
gemma-test | (EngineCore pid=177) ERROR 10-30 12:01:37 [core.py:1510] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
gemma-test | (EngineCore pid=177) ERROR 10-30 12:01:37 [core.py:1510] File “/usr/local/lib/python3.12/dist-packages/vllm/model_executor/model_loader/weight_utils.py”, line 1208, in safetensors_weights_iterator
gemma-test | (EngineCore pid=177) ERROR 10-30 12:01:37 [core.py:1510] with safe_open(st_file, framework=“pt”) as f:
gemma-test | (EngineCore pid=177) ERROR 10-30 12:01:37 [core.py:1510] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
gemma-test | (EngineCore pid=177) ERROR 10-30 12:01:37 [core.py:1510] RuntimeError: unable to mmap 49784788364 bytes from file </root/.cache/huggingface/hub/models–google–gemma-4-31B/snapshots/5bbc2fb1c1b2c611d06e3d9f23c170ba21659d89/model-00001-of-00002.safetensors>: Cannot allocate memory (12)
gemma-test | Traceback (most recent call last):
gemma-test | File “/usr/lib/python3.12/multiprocessing/process.py”, line 314, in _bootstrap
gemma-test | self.run()
gemma-test | File “/usr/lib/python3.12/multiprocessing/process.py”, line 108, in run
gemma-test | self._target(*self._args, **self._kwargs)
gemma-test | File “/usr/local/lib/python3.12/dist-packages/vllm/v1/engine/core.py”, line 1514, in run_engine_core
gemma-test | raise e
gemma-test | File “/usr/local/lib/python3.12/dist-packages/vllm/v1/engine/core.py”, line 1482, in run_engine_core
gemma-test | engine_core = EngineCoreProc(*args, engine_index=dp_rank, **kwargs)
gemma-test | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
gemma-test | File “/usr/local/lib/python3.12/dist-packages/vllm/tracing/otel.py”, line 175, in sync_wrapper
gemma-test | return func(*args, **kwargs)
gemma-test | ^^^^^^^^^^^^^^^^^^^^^
gemma-test | File “/usr/local/lib/python3.12/dist-packages/vllm/v1/engine/core.py”, line 1194, in init
gemma-test | super().init(
gemma-test | File “/usr/local/lib/python3.12/dist-packages/vllm/v1/engine/core.py”, line 144, in init
gemma-test | self.model_executor = executor_class(vllm_config)
gemma-test | ^^^^^^^^^^^^^^^^^^^^^^^^^^^
gemma-test | File “/usr/local/lib/python3.12/dist-packages/vllm/tracing/otel.py”, line 175, in sync_wrapper
gemma-test | return func(*args, **kwargs)
gemma-test | ^^^^^^^^^^^^^^^^^^^^^
gemma-test | File “/usr/local/lib/python3.12/dist-packages/vllm/v1/executor/abstract.py”, line 126, in init
gemma-test | self._init_executor()
gemma-test | File “/usr/local/lib/python3.12/dist-packages/vllm/v1/executor/uniproc_executor.py”, line 74, in _init_executor
gemma-test | self.driver_worker.load_model()
gemma-test | File “/usr/local/lib/python3.12/dist-packages/vllm/v1/worker/gpu_worker.py”, line 594, in load_model
gemma-test | self.model_runner.load_model(load_dummy_weights=load_dummy_weights)
gemma-test | File “/usr/local/lib/python3.12/dist-packages/vllm/v1/worker/gpu/model_runner.py”, line 396, in load_model
gemma-test | self.model = model or model_loader.load_model(
gemma-test | ^^^^^^^^^^^^^^^^^^^^^^^^
gemma-test | File “/usr/local/lib/python3.12/dist-packages/vllm/tracing/otel.py”, line 175, in sync_wrapper
gemma-test | return func(*args, **kwargs)
gemma-test | ^^^^^^^^^^^^^^^^^^^^^
gemma-test | File “/usr/local/lib/python3.12/dist-packages/vllm/model_executor/model_loader/base_loader.py”, line 88, in load_model
gemma-test | self.load_weights(model, model_config)
gemma-test | File “/usr/local/lib/python3.12/dist-packages/vllm/tracing/otel.py”, line 175, in sync_wrapper
gemma-test | return func(*args, **kwargs)
gemma-test | ^^^^^^^^^^^^^^^^^^^^^
gemma-test | File “/usr/local/lib/python3.12/dist-packages/vllm/model_executor/model_loader/default_loader.py”, line 498, in load_weights
gemma-test | loaded_weights = model.load_weights(self.get_all_weights(model_config, model))
gemma-test | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
gemma-test | File “/usr/local/lib/python3.12/dist-packages/vllm/model_executor/models/gemma4_mm.py”, line 2228, in load_weights
gemma-test | return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
gemma-test | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
gemma-test | File “/usr/local/lib/python3.12/dist-packages/vllm/model_executor/model_loader/reload/torchao_decorator.py”, line 52, in patched_model_load_weights
gemma-test | return original_load_weights(self, *args, **kwargs)
gemma-test | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
gemma-test | File “/usr/local/lib/python3.12/dist-packages/vllm/model_executor/models/utils.py”, line 476, in load_weights
gemma-test | autoloaded_weights = set(self._load_module(“”, self.module, weights))
gemma-test | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
gemma-test | File “/usr/local/lib/python3.12/dist-packages/vllm/model_executor/models/utils.py”, line 409, in _load_module
gemma-test | for child_prefix, child_weights in self._groupby_prefix(weights):
gemma-test | File “/usr/local/lib/python3.12/dist-packages/vllm/model_executor/models/utils.py”, line 272, in _groupby_prefix
gemma-test | for prefix, group in itertools.groupby(weights_by_parts, key=lambda x: x[0][0]):
gemma-test | File “/usr/local/lib/python3.12/dist-packages/vllm/model_executor/models/utils.py”, line 267, in
gemma-test | weights_by_parts = (
gemma-test | ^
gemma-test | File “/usr/local/lib/python3.12/dist-packages/vllm/model_executor/models/utils.py”, line 483, in _filter_skipped
gemma-test | for name, weight in weights:
gemma-test | File “/usr/local/lib/python3.12/dist-packages/vllm/model_executor/models/utils.py”, line 150, in apply
gemma-test | for name, data in weights:
gemma-test | File “/usr/local/lib/python3.12/dist-packages/vllm/model_executor/model_loader/default_loader.py”, line 380, in get_all_weights
gemma-test | yield from self._get_weights_iterator(primary_weights)
gemma-test | File “/usr/local/lib/python3.12/dist-packages/vllm/model_executor/model_loader/default_loader.py”, line 365, in
gemma-test | return ((source.prefix + name, tensor) for (name, tensor) in weights_iterator)
gemma-test | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
gemma-test | File “/usr/local/lib/python3.12/dist-packages/vllm/model_executor/model_loader/weight_utils.py”, line 1208, in safetensors_weights_iterator
gemma-test | with safe_open(st_file, framework=“pt”) as f:
gemma-test | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
gemma-test | RuntimeError: unable to mmap 49784788364 bytes from file </root/.cache/huggingface/hub/models–google–gemma-4-31B/snapshots/5bbc2fb1c1b2c611d06e3d9f23c170ba21659d89/model-00001-of-00002.safetensors>: Cannot allocate memory (12)
gemma-test | [rank0]:[W1030 12:01:37.797213023 ProcessGroupNCCL.cpp:1624] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see Redirecting… (function operator())
gemma-test | Traceback (most recent call last):
gemma-test | File “/usr/local/bin/vllm”, line 10, in
gemma-test | sys.exit(main())
gemma-test | ^^^^^^
gemma-test | File “/usr/local/lib/python3.12/dist-packages/vllm/entrypoints/cli/main.py”, line 120, in main
gemma-test | args.dispatch_function(args)
gemma-test | File “/usr/local/lib/python3.12/dist-packages/vllm/entrypoints/cli/serve.py”, line 156, in cmd
gemma-test | uvloop.run(run_server(args))
gemma-test | File “/usr/local/lib/python3.12/dist-packages/uvloop/init.py”, line 96, in run
gemma-test | return __asyncio.run(
gemma-test | ^^^^^^^^^^^^^^
gemma-test | File “/usr/lib/python3.12/asyncio/runners.py”, line 194, in run
gemma-test | return runner.run(main)
gemma-test | ^^^^^^^^^^^^^^^^
gemma-test | File “/usr/lib/python3.12/asyncio/runners.py”, line 118, in run
gemma-test | return self._loop.run_until_complete(task)
gemma-test | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
gemma-test | File “uvloop/loop.pyx”, line 1515, in uvloop.loop.Loop.run_until_complete
gemma-test | File “/usr/local/lib/python3.12/dist-packages/uvloop/init.py”, line 48, in wrapper
gemma-test | return await main
gemma-test | ^^^^^^^^^^
gemma-test | File “/usr/local/lib/python3.12/dist-packages/vllm/entrypoints/launchers/api_server/entry.py”, line 177, in run_server
gemma-test | await run_server_worker(listen_address, sock, args, **uvicorn_kwargs)
gemma-test | File “/usr/local/lib/python3.12/dist-packages/vllm/entrypoints/launchers/api_server/entry.py”, line 195, in run_server_worker
gemma-test | async with build_async_engine_client(
gemma-test | File “/usr/lib/python3.12/contextlib.py”, line 210, in aenter
gemma-test | return await anext(self.gen)
gemma-test | ^^^^^^^^^^^^^^^^^^^^^
gemma-test | File “/usr/local/lib/python3.12/dist-packages/vllm/entrypoints/launchers/api_server/entry.py”, line 61, in build_async_engine_client
gemma-test | async with build_async_engine_client_from_engine_args(
gemma-test | File “/usr/lib/python3.12/contextlib.py”, line 210, in aenter
gemma-test | return await anext(self.gen)
gemma-test | ^^^^^^^^^^^^^^^^^^^^^
gemma-test | File “/usr/local/lib/python3.12/dist-packages/vllm/entrypoints/launchers/api_server/entry.py”, line 95, in build_async_engine_client_from_engine_args
gemma-test | async_llm = AsyncLLM.from_vllm_config(
gemma-test | ^^^^^^^^^^^^^^^^^^^^^^^^^^
gemma-test | File “/usr/local/lib/python3.12/dist-packages/vllm/v1/engine/async_llm.py”, line 255, in from_vllm_config
gemma-test | return cls(
gemma-test | ^^^^
gemma-test | File “/usr/local/lib/python3.12/dist-packages/vllm/v1/engine/async_llm.py”, line 184, in init
gemma-test | self.engine_core = EngineCoreClient.make_async_mp_client(
gemma-test | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
gemma-test | File “/usr/local/lib/python3.12/dist-packages/vllm/tracing/otel.py”, line 175, in sync_wrapper
gemma-test | return func(*args, **kwargs)
gemma-test | ^^^^^^^^^^^^^^^^^^^^^
gemma-test | File “/usr/local/lib/python3.12/dist-packages/vllm/v1/engine/core_client.py”, line 167, in make_async_mp_client
gemma-test | return AsyncMPClient(
gemma-test | ^^^^^^^^^^^^^^
gemma-test | File “/usr/local/lib/python3.12/dist-packages/vllm/tracing/otel.py”, line 175, in sync_wrapper
gemma-test | return func(*args, **kwargs)
gemma-test | ^^^^^^^^^^^^^^^^^^^^^
gemma-test | File “/usr/local/lib/python3.12/dist-packages/vllm/v1/engine/core_client.py”, line 1120, in init
gemma-test | super().init(
gemma-test | File “/usr/local/lib/python3.12/dist-packages/vllm/v1/engine/core_client.py”, line 682, in init
gemma-test | with launch_core_engines(
gemma-test | File “/usr/lib/python3.12/contextlib.py”, line 144, in exit
gemma-test | next(self.gen)
gemma-test | File “/usr/local/lib/python3.12/dist-packages/vllm/v1/engine/utils.py”, line 1351, in launch_core_engines
gemma-test | wait_for_engine_startup(
gemma-test | File “/usr/local/lib/python3.12/dist-packages/vllm/v1/engine/utils.py”, line 1431, in wait_for_engine_startup
gemma-test | raise RuntimeError(
gemma-test | RuntimeError: Engine core initialization failed. See root cause above. Failed core proc(s): {}
Gracefully Stopping… press Ctrl+C again to force
Container gemma-test Stopping
gemma-test | Exception ignored in: <module ‘threading’ from ‘/usr/lib/python3.12/threading.py’>
gemma-test | Traceback (most recent call last):
gemma-test | File “/usr/lib/python3.12/threading.py”, line 1592, in _shutdown
gemma-test | atexit_call()
gemma-test | File “/usr/lib/python3.12/concurrent/futures/thread.py”, line 31, in _python_exit
gemma-test | t.join()
gemma-test | File “/usr/lib/python3.12/threading.py”, line 1147, in join
gemma-test | self._wait_for_tstate_lock()
gemma-test | File “/usr/lib/python3.12/threading.py”, line 1167, in _wait_for_tstate_lock
gemma-test | if lock.acquire(block, timeout):
gemma-test | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
gemma-test | File “/usr/local/lib/python3.12/dist-packages/vllm/entrypoints/launchers/api_server/entry.py”, line 172, in _interrupt_init
gemma-test | raise KeyboardInterrupt(“terminated”)
gemma-test | KeyboardInterrupt: terminated
^CContainer gemma-test Killing
Container gemma-test Killed
Container gemma-test Stopped
root@gpu-prod:/home/neo/ranjith/rnd# cat gemma-4.yaml
services:
gemma-test:
image: vllm/vllm-openai:v0.32.0
container_name: gemma-test
runtime: nvidia
ipc: host
environment:
NVIDIA_VISIBLE_DEVICES: ‘1’
volumes:
- /root/.cache/huggingface:/root/.cache/huggingface
- /dev/null:/etc/ld.so.conf.d/00-cuda-compat.conf
ports: - 9888:8000
command: |-
/root/.cache/huggingface/hub/models–google–gemma-4-31B/snapshots/5bbc2fb1c1b2c611d06e3d9f23c170ba21659d89
–served-model-name gemma-test
–gpu-memory-utilization 0.8
–max-model-len 16304
–max-num-batched-tokens 4096
–mm-processor-cache-gb 0
healthcheck:
test: - CMD-SHELL
- curl -f http://localhost:8000/health || exit 1
interval: 30s
timeout: 10s
retries: 3
start_period: 450s