151 lines
20 KiB
Plaintext
151 lines
20 KiB
Plaintext
[DEBUG] conda_env=VLLM
|
||
[DEBUG] CONDA_PREFIX=/home/huangfukk/miniconda3/envs/VLLM
|
||
[DEBUG] which python: /home/huangfukk/miniconda3/envs/VLLM/bin/python
|
||
[DEBUG] which vllm: /home/huangfukk/miniconda3/envs/VLLM/bin/vllm
|
||
0.13.0
|
||
[DEBUG] python: 3.12.12 | packaged by Anaconda, Inc. | (main, Oct 21 2025, 20:16:04) [GCC 11.2.0]
|
||
[DEBUG] torch: 2.9.0+cu128 cuda: 12.8 available: True
|
||
[0;36m(APIServer pid=43538)[0;0m INFO 01-20 16:25:05 [api_server.py:1351] vLLM API server version 0.13.0
|
||
[0;36m(APIServer pid=43538)[0;0m INFO 01-20 16:25:05 [utils.py:253] non-default args: {'model_tag': '/home/huangfukk/models/AWQ/Qwen3-4B-Instruct-2507', 'host': '0.0.0.0', 'port': 8081, 'uvicorn_log_level': 'debug', 'chat_template': '/home/huangfukk/DronePlanning/tools/vllm_templates/qwen3_xml_tool.jinja', 'chat_template_content_format': 'string', 'enable_auto_tool_choice': True, 'tool_call_parser': 'qwen3_xml', 'model': '/home/huangfukk/models/AWQ/Qwen3-4B-Instruct-2507', 'trust_remote_code': True, 'max_model_len': 16384, 'served_model_name': ['local-model'], 'max_parallel_loading_workers': 4, 'gpu_memory_utilization': 0.5, 'max_num_seqs': 1}
|
||
[0;36m(APIServer pid=43538)[0;0m The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored.
|
||
[0;36m(APIServer pid=43538)[0;0m INFO 01-20 16:25:05 [model.py:514] Resolved architecture: Qwen3ForCausalLM
|
||
[0;36m(APIServer pid=43538)[0;0m INFO 01-20 16:25:05 [model.py:1661] Using max model len 16384
|
||
[0;36m(APIServer pid=43538)[0;0m INFO 01-20 16:25:05 [awq_marlin.py:162] The model is convertible to awq_marlin during runtime. Using awq_marlin kernel.
|
||
[0;36m(APIServer pid=43538)[0;0m WARNING 01-20 16:25:05 [parallel.py:595] max_parallel_loading_workers is currently not supported and will be ignored.
|
||
[0;36m(APIServer pid=43538)[0;0m INFO 01-20 16:25:05 [scheduler.py:230] Chunked prefill is enabled with max_num_batched_tokens=2048.
|
||
[0;36m(APIServer pid=43538)[0;0m WARNING 01-20 16:25:06 [interface.py:465] Using 'pin_memory=False' as WSL is detected. This may slow down the performance.
|
||
[0;36m(EngineCore_DP0 pid=43785)[0;0m INFO 01-20 16:25:10 [core.py:93] Initializing a V1 LLM engine (v0.13.0) with config: model='/home/huangfukk/models/AWQ/Qwen3-4B-Instruct-2507', speculative_config=None, tokenizer='/home/huangfukk/models/AWQ/Qwen3-4B-Instruct-2507', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=16384, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, disable_custom_all_reduce=False, quantization=awq_marlin, enforce_eager=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_fallback=False, disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False), seed=0, served_model_name=local-model, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'level': None, 'mode': <CompilationMode.VLLM_COMPILE: 3>, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'splitting_ops': ['vllm::unified_attention', 'vllm::unified_attention_with_output', 'vllm::unified_mla_attention', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::gdn_attention_core', 'vllm::kda_attention', 'vllm::sparse_attn_indexer'], 'compile_mm_encoder': False, 'compile_sizes': [], 'compile_ranges_split_points': [2048], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': <CUDAGraphMode.FULL_AND_PIECEWISE: (2, 1)>, 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'eliminate_noops': True, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False}, 'max_cudagraph_capture_size': 2, 'dynamic_shapes_config': {'type': <DynamicShapesType.BACKED: 'backed'>, 'evaluate_guards': False}, 'local_cache_dir': None}
|
||
[0;36m(EngineCore_DP0 pid=43785)[0;0m INFO 01-20 16:25:11 [parallel_state.py:1203] world_size=1 rank=0 local_rank=0 distributed_init_method=tcp://192.168.31.62:43504 backend=nccl
|
||
[W120 16:25:21.543804205 socket.cpp:209] [c10d] The hostname of the client socket cannot be retrieved. err=-3
|
||
[0;36m(EngineCore_DP0 pid=43785)[0;0m INFO 01-20 16:25:21 [parallel_state.py:1411] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank 0
|
||
[0;36m(EngineCore_DP0 pid=43785)[0;0m WARNING 01-20 16:25:21 [interface.py:465] Using 'pin_memory=False' as WSL is detected. This may slow down the performance.
|
||
[0;36m(EngineCore_DP0 pid=43785)[0;0m INFO 01-20 16:25:22 [gpu_model_runner.py:3562] Starting to load model /home/huangfukk/models/AWQ/Qwen3-4B-Instruct-2507...
|
||
[0;36m(EngineCore_DP0 pid=43785)[0;0m INFO 01-20 16:25:22 [cuda.py:351] Using FLASH_ATTN attention backend out of potential backends: ('FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION')
|
||
[0;36m(EngineCore_DP0 pid=43785)[0;0m
|
||
Loading safetensors checkpoint shards: 0% Completed | 0/1 [00:00<?, ?it/s]
|
||
[0;36m(EngineCore_DP0 pid=43785)[0;0m
|
||
Loading safetensors checkpoint shards: 100% Completed | 1/1 [00:03<00:00, 3.71s/it]
|
||
[0;36m(EngineCore_DP0 pid=43785)[0;0m
|
||
Loading safetensors checkpoint shards: 100% Completed | 1/1 [00:03<00:00, 3.71s/it]
|
||
[0;36m(EngineCore_DP0 pid=43785)[0;0m
|
||
[0;36m(EngineCore_DP0 pid=43785)[0;0m INFO 01-20 16:25:26 [default_loader.py:308] Loading weights took 2.68 seconds
|
||
[0;36m(EngineCore_DP0 pid=43785)[0;0m INFO 01-20 16:25:27 [gpu_model_runner.py:3659] Model loading took 2.5630 GiB memory and 3.642749 seconds
|
||
[0;36m(EngineCore_DP0 pid=43785)[0;0m INFO 01-20 16:25:34 [backends.py:643] Using cache directory: /home/huangfukk/.cache/vllm/torch_compile_cache/d651665fcd/rank_0_0/backbone for vLLM's torch.compile
|
||
[0;36m(EngineCore_DP0 pid=43785)[0;0m INFO 01-20 16:25:34 [backends.py:703] Dynamo bytecode transform time: 7.23 s
|
||
[0;36m(EngineCore_DP0 pid=43785)[0;0m INFO 01-20 16:25:39 [backends.py:226] Directly load the compiled graph(s) for compile range (1, 2048) from the cache, took 1.264 s
|
||
[0;36m(EngineCore_DP0 pid=43785)[0;0m INFO 01-20 16:25:39 [monitor.py:34] torch.compile takes 8.49 s in total
|
||
[0;36m(EngineCore_DP0 pid=43785)[0;0m INFO 01-20 16:25:40 [gpu_worker.py:375] Available KV cache memory: 5.26 GiB
|
||
[0;36m(EngineCore_DP0 pid=43785)[0;0m INFO 01-20 16:25:40 [kv_cache_utils.py:1291] GPU KV cache size: 38,288 tokens
|
||
[0;36m(EngineCore_DP0 pid=43785)[0;0m INFO 01-20 16:25:40 [kv_cache_utils.py:1296] Maximum concurrency for 16,384 tokens per request: 2.34x
|
||
[0;36m(EngineCore_DP0 pid=43785)[0;0m
|
||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 0%| | 0/2 [00:00<?, ?it/s]
|
||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 100%|██████████| 2/2 [00:00<00:00, 21.01it/s]
|
||
[0;36m(EngineCore_DP0 pid=43785)[0;0m
|
||
Capturing CUDA graphs (decode, FULL): 0%| | 0/1 [00:00<?, ?it/s]
|
||
Capturing CUDA graphs (decode, FULL): 100%|██████████| 1/1 [00:00<00:00, 12.71it/s]
|
||
[0;36m(EngineCore_DP0 pid=43785)[0;0m INFO 01-20 16:25:40 [gpu_model_runner.py:4587] Graph capturing finished in 1 secs, took 0.02 GiB
|
||
[0;36m(EngineCore_DP0 pid=43785)[0;0m INFO 01-20 16:25:40 [core.py:259] init engine (profile, create kv cache, warmup model) took 13.61 seconds
|
||
[0;36m(APIServer pid=43538)[0;0m INFO 01-20 16:25:41 [api_server.py:1099] Supported tasks: ['generate']
|
||
[0;36m(APIServer pid=43538)[0;0m WARNING 01-20 16:25:42 [utils.py:312] Using supplied chat template: {%- if tools %}
|
||
[0;36m(APIServer pid=43538)[0;0m WARNING 01-20 16:25:42 [utils.py:312] <|im_start|>system
|
||
[0;36m(APIServer pid=43538)[0;0m WARNING 01-20 16:25:42 [utils.py:312] {%- if messages[0].role == 'system' -%}
|
||
[0;36m(APIServer pid=43538)[0;0m WARNING 01-20 16:25:42 [utils.py:312] {{ messages[0].content }}
|
||
[0;36m(APIServer pid=43538)[0;0m WARNING 01-20 16:25:42 [utils.py:312] {%- else -%}
|
||
[0;36m(APIServer pid=43538)[0;0m WARNING 01-20 16:25:42 [utils.py:312] You are a helpful assistant.
|
||
[0;36m(APIServer pid=43538)[0;0m WARNING 01-20 16:25:42 [utils.py:312] {%- endif -%}
|
||
[0;36m(APIServer pid=43538)[0;0m WARNING 01-20 16:25:42 [utils.py:312]
|
||
[0;36m(APIServer pid=43538)[0;0m WARNING 01-20 16:25:42 [utils.py:312] # Tools
|
||
[0;36m(APIServer pid=43538)[0;0m WARNING 01-20 16:25:42 [utils.py:312] You may call one or more functions to assist with the user query.
|
||
[0;36m(APIServer pid=43538)[0;0m WARNING 01-20 16:25:42 [utils.py:312] The function signatures are provided below:
|
||
[0;36m(APIServer pid=43538)[0;0m WARNING 01-20 16:25:42 [utils.py:312] <tools>
|
||
[0;36m(APIServer pid=43538)[0;0m WARNING 01-20 16:25:42 [utils.py:312] {%- for tool in tools %}
|
||
[0;36m(APIServer pid=43538)[0;0m WARNING 01-20 16:25:42 [utils.py:312] {{ tool | tojson }}
|
||
[0;36m(APIServer pid=43538)[0;0m WARNING 01-20 16:25:42 [utils.py:312] {%- endfor %}
|
||
[0;36m(APIServer pid=43538)[0;0m WARNING 01-20 16:25:42 [utils.py:312] </tools>
|
||
[0;36m(APIServer pid=43538)[0;0m WARNING 01-20 16:25:42 [utils.py:312]
|
||
[0;36m(APIServer pid=43538)[0;0m WARNING 01-20 16:25:42 [utils.py:312] When you need to call a tool, respond with XML tags in this exact format:
|
||
[0;36m(APIServer pid=43538)[0;0m WARNING 01-20 16:25:42 [utils.py:312] <tool_call>
|
||
[0;36m(APIServer pid=43538)[0;0m WARNING 01-20 16:25:42 [utils.py:312] <function=tool_name>
|
||
[0;36m(APIServer pid=43538)[0;0m WARNING 01-20 16:25:42 [utils.py:312] <parameter=param1>value1</parameter>
|
||
[0;36m(APIServer pid=43538)[0;0m WARNING 01-20 16:25:42 [utils.py:312] <parameter=param2>value2</parameter>
|
||
[0;36m(APIServer pid=43538)[0;0m WARNING 01-20 16:25:42 [utils.py:312] </function>
|
||
[0;36m(APIServer pid=43538)[0;0m WARNING 01-20 16:25:42 [utils.py:312] </tool_call>
|
||
[0;36m(APIServer pid=43538)[0;0m WARNING 01-20 16:25:42 [utils.py:312] <|im_end|>
|
||
[0;36m(APIServer pid=43538)[0;0m WARNING 01-20 16:25:42 [utils.py:312] {%- else %}
|
||
[0;36m(APIServer pid=43538)[0;0m WARNING 01-20 16:25:42 [utils.py:312] {%- if messages[0].role == 'system' -%}
|
||
[0;36m(APIServer pid=43538)[0;0m WARNING 01-20 16:25:42 [utils.py:312] <|im_start|>system
|
||
[0;36m(APIServer pid=43538)[0;0m WARNING 01-20 16:25:42 [utils.py:312] {{ messages[0].content }}<|im_end|>
|
||
[0;36m(APIServer pid=43538)[0;0m WARNING 01-20 16:25:42 [utils.py:312] {%- endif -%}
|
||
[0;36m(APIServer pid=43538)[0;0m WARNING 01-20 16:25:42 [utils.py:312] {%- endif %}
|
||
[0;36m(APIServer pid=43538)[0;0m WARNING 01-20 16:25:42 [utils.py:312]
|
||
[0;36m(APIServer pid=43538)[0;0m WARNING 01-20 16:25:42 [utils.py:312] {%- for message in messages %}
|
||
[0;36m(APIServer pid=43538)[0;0m WARNING 01-20 16:25:42 [utils.py:312] {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
|
||
[0;36m(APIServer pid=43538)[0;0m WARNING 01-20 16:25:42 [utils.py:312] <|im_start|>{{ message.role }}
|
||
[0;36m(APIServer pid=43538)[0;0m WARNING 01-20 16:25:42 [utils.py:312] {{ message.content }}<|im_end|>
|
||
[0;36m(APIServer pid=43538)[0;0m WARNING 01-20 16:25:42 [utils.py:312] {%- elif message.role == "assistant" %}
|
||
[0;36m(APIServer pid=43538)[0;0m WARNING 01-20 16:25:42 [utils.py:312] <|im_start|>assistant
|
||
[0;36m(APIServer pid=43538)[0;0m WARNING 01-20 16:25:42 [utils.py:312] {%- if message.content %}
|
||
[0;36m(APIServer pid=43538)[0;0m WARNING 01-20 16:25:42 [utils.py:312] {{ message.content }}
|
||
[0;36m(APIServer pid=43538)[0;0m WARNING 01-20 16:25:42 [utils.py:312] {%- endif %}
|
||
[0;36m(APIServer pid=43538)[0;0m WARNING 01-20 16:25:42 [utils.py:312] <|im_end|>
|
||
[0;36m(APIServer pid=43538)[0;0m WARNING 01-20 16:25:42 [utils.py:312] {%- elif message.role == "tool" %}
|
||
[0;36m(APIServer pid=43538)[0;0m WARNING 01-20 16:25:42 [utils.py:312] <|im_start|>user
|
||
[0;36m(APIServer pid=43538)[0;0m WARNING 01-20 16:25:42 [utils.py:312] <tool_response>
|
||
[0;36m(APIServer pid=43538)[0;0m WARNING 01-20 16:25:42 [utils.py:312] {{ message.content }}
|
||
[0;36m(APIServer pid=43538)[0;0m WARNING 01-20 16:25:42 [utils.py:312] </tool_response>
|
||
[0;36m(APIServer pid=43538)[0;0m WARNING 01-20 16:25:42 [utils.py:312] <|im_end|>
|
||
[0;36m(APIServer pid=43538)[0;0m WARNING 01-20 16:25:42 [utils.py:312] {%- endif %}
|
||
[0;36m(APIServer pid=43538)[0;0m WARNING 01-20 16:25:42 [utils.py:312] {%- endfor %}
|
||
[0;36m(APIServer pid=43538)[0;0m WARNING 01-20 16:25:42 [utils.py:312]
|
||
[0;36m(APIServer pid=43538)[0;0m WARNING 01-20 16:25:42 [utils.py:312] {%- if add_generation_prompt %}
|
||
[0;36m(APIServer pid=43538)[0;0m WARNING 01-20 16:25:42 [utils.py:312] <|im_start|>assistant
|
||
[0;36m(APIServer pid=43538)[0;0m WARNING 01-20 16:25:42 [utils.py:312] {%- endif %}
|
||
[0;36m(APIServer pid=43538)[0;0m WARNING 01-20 16:25:42 [utils.py:312]
|
||
[0;36m(APIServer pid=43538)[0;0m WARNING 01-20 16:25:42 [utils.py:312] It is different from official chat template '/home/huangfukk/models/AWQ/Qwen3-4B-Instruct-2507'. This discrepancy may lead to performance degradation.
|
||
[0;36m(APIServer pid=43538)[0;0m WARNING 01-20 16:25:42 [model.py:1487] Default sampling parameters have been overridden by the model's Hugging Face generation config recommended from the model creator. If this is not intended, please relaunch vLLM instance with `--generation-config vllm`.
|
||
[0;36m(APIServer pid=43538)[0;0m INFO 01-20 16:25:42 [serving_responses.py:201] Using default chat sampling params from model: {'temperature': 0.7, 'top_k': 20, 'top_p': 0.8}
|
||
[0;36m(APIServer pid=43538)[0;0m INFO 01-20 16:25:42 [serving_engine.py:270] "auto" tool choice has been enabled.
|
||
[0;36m(APIServer pid=43538)[0;0m INFO 01-20 16:25:42 [serving_engine.py:270] "auto" tool choice has been enabled.
|
||
[0;36m(APIServer pid=43538)[0;0m INFO 01-20 16:25:42 [serving_chat.py:137] Using default chat sampling params from model: {'temperature': 0.7, 'top_k': 20, 'top_p': 0.8}
|
||
[0;36m(APIServer pid=43538)[0;0m INFO 01-20 16:25:42 [serving_completion.py:77] Using default completion sampling params from model: {'temperature': 0.7, 'top_k': 20, 'top_p': 0.8}
|
||
[0;36m(APIServer pid=43538)[0;0m INFO 01-20 16:25:42 [serving_engine.py:270] "auto" tool choice has been enabled.
|
||
[0;36m(APIServer pid=43538)[0;0m INFO 01-20 16:25:42 [serving_chat.py:137] Using default chat sampling params from model: {'temperature': 0.7, 'top_k': 20, 'top_p': 0.8}
|
||
[0;36m(APIServer pid=43538)[0;0m INFO 01-20 16:25:42 [api_server.py:1425] Starting vLLM API server 0 on http://0.0.0.0:8081
|
||
[0;36m(APIServer pid=43538)[0;0m INFO 01-20 16:25:42 [launcher.py:38] Available routes are:
|
||
[0;36m(APIServer pid=43538)[0;0m INFO 01-20 16:25:42 [launcher.py:46] Route: /openapi.json, Methods: GET, HEAD
|
||
[0;36m(APIServer pid=43538)[0;0m INFO 01-20 16:25:42 [launcher.py:46] Route: /docs, Methods: GET, HEAD
|
||
[0;36m(APIServer pid=43538)[0;0m INFO 01-20 16:25:42 [launcher.py:46] Route: /docs/oauth2-redirect, Methods: GET, HEAD
|
||
[0;36m(APIServer pid=43538)[0;0m INFO 01-20 16:25:42 [launcher.py:46] Route: /redoc, Methods: GET, HEAD
|
||
[0;36m(APIServer pid=43538)[0;0m INFO 01-20 16:25:42 [launcher.py:46] Route: /scale_elastic_ep, Methods: POST
|
||
[0;36m(APIServer pid=43538)[0;0m INFO 01-20 16:25:42 [launcher.py:46] Route: /is_scaling_elastic_ep, Methods: POST
|
||
[0;36m(APIServer pid=43538)[0;0m INFO 01-20 16:25:42 [launcher.py:46] Route: /tokenize, Methods: POST
|
||
[0;36m(APIServer pid=43538)[0;0m INFO 01-20 16:25:42 [launcher.py:46] Route: /detokenize, Methods: POST
|
||
[0;36m(APIServer pid=43538)[0;0m INFO 01-20 16:25:42 [launcher.py:46] Route: /inference/v1/generate, Methods: POST
|
||
[0;36m(APIServer pid=43538)[0;0m INFO 01-20 16:25:42 [launcher.py:46] Route: /pause, Methods: POST
|
||
[0;36m(APIServer pid=43538)[0;0m INFO 01-20 16:25:42 [launcher.py:46] Route: /resume, Methods: POST
|
||
[0;36m(APIServer pid=43538)[0;0m INFO 01-20 16:25:42 [launcher.py:46] Route: /is_paused, Methods: GET
|
||
[0;36m(APIServer pid=43538)[0;0m INFO 01-20 16:25:42 [launcher.py:46] Route: /metrics, Methods: GET
|
||
[0;36m(APIServer pid=43538)[0;0m INFO 01-20 16:25:42 [launcher.py:46] Route: /health, Methods: GET
|
||
[0;36m(APIServer pid=43538)[0;0m INFO 01-20 16:25:42 [launcher.py:46] Route: /load, Methods: GET
|
||
[0;36m(APIServer pid=43538)[0;0m INFO 01-20 16:25:42 [launcher.py:46] Route: /v1/models, Methods: GET
|
||
[0;36m(APIServer pid=43538)[0;0m INFO 01-20 16:25:42 [launcher.py:46] Route: /version, Methods: GET
|
||
[0;36m(APIServer pid=43538)[0;0m INFO 01-20 16:25:42 [launcher.py:46] Route: /v1/responses, Methods: POST
|
||
[0;36m(APIServer pid=43538)[0;0m INFO 01-20 16:25:42 [launcher.py:46] Route: /v1/responses/{response_id}, Methods: GET
|
||
[0;36m(APIServer pid=43538)[0;0m INFO 01-20 16:25:42 [launcher.py:46] Route: /v1/responses/{response_id}/cancel, Methods: POST
|
||
[0;36m(APIServer pid=43538)[0;0m INFO 01-20 16:25:42 [launcher.py:46] Route: /v1/messages, Methods: POST
|
||
[0;36m(APIServer pid=43538)[0;0m INFO 01-20 16:25:42 [launcher.py:46] Route: /v1/chat/completions, Methods: POST
|
||
[0;36m(APIServer pid=43538)[0;0m INFO 01-20 16:25:42 [launcher.py:46] Route: /v1/completions, Methods: POST
|
||
[0;36m(APIServer pid=43538)[0;0m INFO 01-20 16:25:42 [launcher.py:46] Route: /v1/audio/transcriptions, Methods: POST
|
||
[0;36m(APIServer pid=43538)[0;0m INFO 01-20 16:25:42 [launcher.py:46] Route: /v1/audio/translations, Methods: POST
|
||
[0;36m(APIServer pid=43538)[0;0m INFO 01-20 16:25:42 [launcher.py:46] Route: /ping, Methods: GET
|
||
[0;36m(APIServer pid=43538)[0;0m INFO 01-20 16:25:42 [launcher.py:46] Route: /ping, Methods: POST
|
||
[0;36m(APIServer pid=43538)[0;0m INFO 01-20 16:25:42 [launcher.py:46] Route: /invocations, Methods: POST
|
||
[0;36m(APIServer pid=43538)[0;0m INFO 01-20 16:25:42 [launcher.py:46] Route: /classify, Methods: POST
|
||
[0;36m(APIServer pid=43538)[0;0m INFO 01-20 16:25:42 [launcher.py:46] Route: /v1/embeddings, Methods: POST
|
||
[0;36m(APIServer pid=43538)[0;0m INFO 01-20 16:25:42 [launcher.py:46] Route: /score, Methods: POST
|
||
[0;36m(APIServer pid=43538)[0;0m INFO 01-20 16:25:42 [launcher.py:46] Route: /v1/score, Methods: POST
|
||
[0;36m(APIServer pid=43538)[0;0m INFO 01-20 16:25:42 [launcher.py:46] Route: /rerank, Methods: POST
|
||
[0;36m(APIServer pid=43538)[0;0m INFO 01-20 16:25:42 [launcher.py:46] Route: /v1/rerank, Methods: POST
|
||
[0;36m(APIServer pid=43538)[0;0m INFO 01-20 16:25:42 [launcher.py:46] Route: /v2/rerank, Methods: POST
|
||
[0;36m(APIServer pid=43538)[0;0m INFO 01-20 16:25:42 [launcher.py:46] Route: /pooling, Methods: POST
|
||
[0;36m(APIServer pid=43538)[0;0m INFO: Started server process [43538]
|
||
[0;36m(APIServer pid=43538)[0;0m INFO: Waiting for application startup.
|
||
[0;36m(APIServer pid=43538)[0;0m INFO: Application startup complete.
|
||
[0;36m(APIServer pid=43538)[0;0m INFO: 127.0.0.1:46791 - "GET /health HTTP/1.1" 200 OK
|
||
[0;36m(APIServer pid=43538)[0;0m INFO: 127.0.0.1:43619 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||
[0;36m(APIServer pid=43538)[0;0m INFO 01-21 20:22:50 [loggers.py:248] Engine 000: Avg prompt throughput: 168.4 tokens/s, Avg generation throughput: 0.6 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 0.0%
|
||
[0;36m(APIServer pid=43538)[0;0m INFO 01-21 20:23:00 [loggers.py:248] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 0.0 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 0.0%
|