1、vllm 环境安装、运行

conda create -n vllm python=3.10

conda activate vllm

pip install vllm==0.8.5

# 遇到报错的话，将transformer的版本降低到4.57试试
vllm serve /root/autodl-tmp/llm/Qwen/Qwen3-0.6B

报错内容：Qwen2Tokenizer has no attribute all_special_tokens_extended. Did you mean: 'num_special_tokens_to_add'?

# 服务端启用，客户端用openai接口进行调用

# 使用openai的API风格调用本地模型
from openai import OpenAI

client = OpenAI(base_url="http://localhost:8000/v1/",api_key="suibianxie")

chat_completion = client.chat.completions.create(
    messages=[{"role":"user","content":"介绍下你自己"}],model="/root/autodl-tmp/llm/Qwen/Qwen3-0.6B"
)
print(chat_completion.choices[0])

2、lmdeploy 环境安装、运行

conda create -n lmdeploy python=3.10

source activate lmdeploy

pip install lmdeploy

lmdeploy serve api_server /root/autodl-tmp/llm/Qwen/Qwen3-0.6B

# 服务端启用，客户端用openai接口进行调用

# 使用openai的API风格调用本地模型

from openai import OpenAI
client = OpenAI(base_url="http://localhost:23333/v1", api_key="YOUR_KEY")
response = client.chat.completions.create(model="/root/autodl-tmp/llm/Qwen/Qwen3-0.6B", messages=
[{"role":"user", "content":"介绍下深度学习"}])

print(response)